{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import checklist\n",
    "import spacy\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import checklist.editor\n",
    "import checklist.text_generation\n",
    "from checklist.mft import Mft\n",
    "from checklist.inv_dir import Inv, Dir\n",
    "from checklist.expect import Expect\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tg = checklist.text_generation.TextGenerator(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from checklist.pred_wrapper import PredictorWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/marcotcr/work/ml-tests/')\n",
    "from mltests import model_wrapper\n",
    "sentiment = model_wrapper.ModelWrapper()\n",
    "pp = PredictorWrapper.wrap_softmax(sentiment.predict_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tg.unmask_multiple(['I really <mask> the pilot.', 'I really <mask> the flight.'], metric='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 553,
   "metadata": {},
   "outputs": [],
   "source": [
    "editor.__init__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 605,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['male', 'female', 'first_name', 'first_pronoun', 'last_name', 'country', 'nationality', 'city', 'religion', 'religion_adj', 'sexual_adj'])"
      ]
     },
     "execution_count": 605,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [],
   "source": [
    "editor.add_lexicon('pos', pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 615,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.78  This is an interesting flight.\n",
      "  9.31  This is an fun flight.\n",
      "  9.31  This is an scenic flight.\n",
      "  9.07  This is an good flight.\n",
      "  8.94  This is an simplified flight.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['interesting',\n",
       " 'fun',\n",
       " 'scenic',\n",
       " 'good',\n",
       " 'simplified',\n",
       " 'beautiful',\n",
       " 'classic',\n",
       " 'modern',\n",
       " 'successful',\n",
       " 'amazing',\n",
       " 'perfect',\n",
       " 'easy',\n",
       " 'free',\n",
       " 'recovery',\n",
       " 'improved',\n",
       " 'love',\n",
       " 'restored',\n",
       " 'great',\n",
       " 'famous',\n",
       " 'exciting',\n",
       " 'fast',\n",
       " 'popular',\n",
       " 'incredible',\n",
       " 'clean',\n",
       " 'awesome',\n",
       " 'smooth',\n",
       " 'ideal',\n",
       " 'correct',\n",
       " 'optimistic',\n",
       " 'support',\n",
       " 'smoother',\n",
       " 'authentic',\n",
       " 'faster',\n",
       " 'important',\n",
       " 'memorable',\n",
       " 'spectacular',\n",
       " 'nice',\n",
       " 'realistic',\n",
       " 'clear',\n",
       " 'spontaneous',\n",
       " 'fantastic',\n",
       " 'excellent',\n",
       " 'fascinating',\n",
       " 'optimal',\n",
       " 'hot',\n",
       " 'lovely',\n",
       " 'supported',\n",
       " 'heroic',\n",
       " 'efficient',\n",
       " 'straightforward',\n",
       " 'hilarious',\n",
       " 'work',\n",
       " 'pleasant',\n",
       " 'better',\n",
       " 'enhanced',\n",
       " 'solid',\n",
       " 'romantic',\n",
       " 'hopeful',\n",
       " 'safe',\n",
       " 'cleaner',\n",
       " 'exemplary',\n",
       " 'happy',\n",
       " 'extraordinary',\n",
       " 'legendary',\n",
       " 'progress',\n",
       " 'stable',\n",
       " 'relief',\n",
       " 'enjoyable',\n",
       " 'sweet',\n",
       " 'proper',\n",
       " 'friendly',\n",
       " 'silent',\n",
       " 'decent',\n",
       " 'thrilling',\n",
       " 'calm',\n",
       " 'impressive',\n",
       " 'intimate',\n",
       " 'autonomous',\n",
       " 'quiet',\n",
       " 'available',\n",
       " 'accomplished',\n",
       " 'wonderful',\n",
       " 'cool',\n",
       " 'reasonable',\n",
       " 'easier',\n",
       " 'surreal',\n",
       " 'pretty',\n",
       " 'competitive',\n",
       " 'inspiring',\n",
       " 'proven',\n",
       " 'secure',\n",
       " 'fancy',\n",
       " 'peaceful',\n",
       " 'significant',\n",
       " 'humorous',\n",
       " 'intuitive',\n",
       " 'responsive',\n",
       " 'rapid',\n",
       " 'remarkable',\n",
       " 'accurate',\n",
       " 'lucky',\n",
       " 'stunning',\n",
       " 'breathtaking',\n",
       " 'ambitious',\n",
       " 'encouraging',\n",
       " 'intelligence',\n",
       " 'gorgeous',\n",
       " 'advanced',\n",
       " 'triumphant',\n",
       " 'futuristic',\n",
       " 'graceful',\n",
       " 'amusing',\n",
       " 'upgraded',\n",
       " 'powerful',\n",
       " 'favorite',\n",
       " 'appropriate',\n",
       " 'dedicated',\n",
       " 'recommended',\n",
       " 'glorious',\n",
       " 'comfortable',\n",
       " 'quieter',\n",
       " 'survival',\n",
       " 'precise',\n",
       " 'top',\n",
       " 'pure',\n",
       " 'stellar',\n",
       " 'joy',\n",
       " 'reliable',\n",
       " 'super',\n",
       " 'warm',\n",
       " 'elegant',\n",
       " 'genuine',\n",
       " 'streamlined',\n",
       " 'winning',\n",
       " 'meaningful',\n",
       " 'brilliant',\n",
       " 'positive',\n",
       " 'proud',\n",
       " 'productive',\n",
       " 'outstanding',\n",
       " 'adventurous',\n",
       " 'delightful',\n",
       " 'sexy',\n",
       " 'instrumental',\n",
       " 'excited',\n",
       " 'quicker',\n",
       " 'effective',\n",
       " 'unforgettable',\n",
       " 'fresh',\n",
       " 'best',\n",
       " 'welcome',\n",
       " 'unaffected',\n",
       " 'celebration',\n",
       " 'promising',\n",
       " 'simpler',\n",
       " 'complementary',\n",
       " 'balanced',\n",
       " 'lead',\n",
       " 'inspirational',\n",
       " 'supporting',\n",
       " 'revolutionary',\n",
       " 'noteworthy',\n",
       " 'astonishing',\n",
       " 'mature',\n",
       " 'seamless',\n",
       " 'right',\n",
       " 'relaxed',\n",
       " 'inexpensive',\n",
       " 'magical',\n",
       " 'dynamic',\n",
       " 'happier',\n",
       " 'fair',\n",
       " 'angel',\n",
       " 'tranquil',\n",
       " 'like',\n",
       " 'confident',\n",
       " 'confidence',\n",
       " 'convenient',\n",
       " 'cheaper',\n",
       " 'intriguing',\n",
       " 'affordable',\n",
       " 'phenomenal',\n",
       " 'speedy',\n",
       " 'usable',\n",
       " 'pleasure',\n",
       " 'complimentary',\n",
       " 'convincing',\n",
       " 'superb',\n",
       " 'inspiration',\n",
       " 'terrific',\n",
       " 'privileged',\n",
       " 'passionate',\n",
       " 'entertaining',\n",
       " 'fine',\n",
       " 'celebrated',\n",
       " 'improvement',\n",
       " 'loved',\n",
       " 'useful',\n",
       " 'flawless',\n",
       " 'glamorous',\n",
       " 'bright',\n",
       " 'attractive',\n",
       " 'cooperative',\n",
       " 'steady',\n",
       " 'clearer',\n",
       " 'success',\n",
       " 'lighter',\n",
       " 'homage',\n",
       " 'sensational',\n",
       " 'bonus',\n",
       " 'exquisite',\n",
       " 'consistent',\n",
       " 'master',\n",
       " 'survivor',\n",
       " 'innocuous',\n",
       " 'heavenly',\n",
       " 'enthusiast',\n",
       " 'personalized',\n",
       " 'coherent',\n",
       " 'economical',\n",
       " 'festive',\n",
       " 'magnificent',\n",
       " 'delicious',\n",
       " 'exceptional',\n",
       " 'proving',\n",
       " 'striking',\n",
       " 'integrated',\n",
       " 'recover',\n",
       " 'majestic',\n",
       " 'understandable',\n",
       " 'humble',\n",
       " 'enthusiastic',\n",
       " 'honest',\n",
       " 'continuity',\n",
       " 'cute',\n",
       " 'colorful',\n",
       " 'appreciated',\n",
       " 'upbeat',\n",
       " 'joyful',\n",
       " 'accessible',\n",
       " 'progressive',\n",
       " 'tough',\n",
       " 'precious',\n",
       " 'brave',\n",
       " 'logical',\n",
       " 'guidance',\n",
       " 'promised',\n",
       " 'suitable',\n",
       " 'diplomatic',\n",
       " 'healthy',\n",
       " 'soft',\n",
       " 'innovative',\n",
       " 'faithful',\n",
       " 'tender',\n",
       " 'spiritual',\n",
       " 'valuable',\n",
       " 'lively',\n",
       " 'honor',\n",
       " 'refined',\n",
       " 'sharp',\n",
       " 'swift',\n",
       " 'worked',\n",
       " 'lean',\n",
       " 'revival',\n",
       " 'timely',\n",
       " 'neat',\n",
       " 'playful',\n",
       " 'intricate',\n",
       " 'magic',\n",
       " 'works',\n",
       " 'fabulous',\n",
       " 'patient',\n",
       " 'creative',\n",
       " 'variety',\n",
       " 'subsidized',\n",
       " 'sensible',\n",
       " 'comprehensive',\n",
       " 'ready',\n",
       " 'fastest',\n",
       " 'luxurious',\n",
       " 'comfort',\n",
       " 'harmless',\n",
       " 'well',\n",
       " 'charming',\n",
       " 'approval',\n",
       " 'marvelous',\n",
       " 'nicer',\n",
       " 'selective',\n",
       " 'astounding',\n",
       " 'expansive',\n",
       " 'worthwhile',\n",
       " 'integral',\n",
       " 'improving',\n",
       " 'wise',\n",
       " 'respectable',\n",
       " 'helpful',\n",
       " 'rational',\n",
       " 'concise',\n",
       " 'honored',\n",
       " 'reward',\n",
       " 'bullish',\n",
       " 'trophy',\n",
       " 'unlimited',\n",
       " 'calming',\n",
       " 'victory',\n",
       " 'intelligent',\n",
       " 'transparent',\n",
       " 'loving',\n",
       " 'spacious',\n",
       " 'adorable',\n",
       " 'crisp',\n",
       " 'motivated',\n",
       " 'delicate',\n",
       " 'succeeding',\n",
       " 'polite',\n",
       " 'victorious',\n",
       " 'energetic',\n",
       " 'tempting',\n",
       " 'refreshing',\n",
       " 'grand',\n",
       " 'peace',\n",
       " 'rewarding',\n",
       " 'spirited',\n",
       " 'modest',\n",
       " 'strong',\n",
       " 'slick',\n",
       " 'supportive',\n",
       " 'insightful',\n",
       " 'defeated',\n",
       " 'redemption',\n",
       " 'smart',\n",
       " 'gentle',\n",
       " 'illuminating',\n",
       " 'stability',\n",
       " 'won',\n",
       " 'superior',\n",
       " 'brisk',\n",
       " 'appealing',\n",
       " 'beloved',\n",
       " 'poetic',\n",
       " 'qualified',\n",
       " 'educated',\n",
       " 'luxury',\n",
       " 'rich',\n",
       " 'divine',\n",
       " 'decisive',\n",
       " 'ecstatic',\n",
       " 'cheerful',\n",
       " 'freedom',\n",
       " 'fortunate',\n",
       " 'constructive',\n",
       " 'beauty',\n",
       " 'fiery',\n",
       " 'hero',\n",
       " 'sophisticated',\n",
       " 'grace',\n",
       " 'distinguished',\n",
       " 'dawn',\n",
       " 'compatible',\n",
       " 'righteous',\n",
       " 'respectful',\n",
       " 'manageable',\n",
       " 'passion',\n",
       " 'groundbreaking',\n",
       " 'excitement',\n",
       " 'mercy',\n",
       " 'sustainable',\n",
       " 'polished',\n",
       " 'thankful',\n",
       " 'leading',\n",
       " 'liberty',\n",
       " 'daring',\n",
       " 'patriotic',\n",
       " 'refreshed',\n",
       " 'vivid',\n",
       " 'capable',\n",
       " 'benefit',\n",
       " 'prestige',\n",
       " 'hail',\n",
       " 'cleared',\n",
       " 'miraculous',\n",
       " 'feasible',\n",
       " 'advantage',\n",
       " 'reconciliation',\n",
       " 'preferable',\n",
       " 'miracle',\n",
       " 'win',\n",
       " 'worth',\n",
       " 'worthy',\n",
       " 'portable',\n",
       " 'pinnacle',\n",
       " 'meticulous',\n",
       " 'sane',\n",
       " 'convenience',\n",
       " 'blockbuster',\n",
       " 'improve',\n",
       " 'sensitive',\n",
       " 'favorable',\n",
       " 'courageous',\n",
       " 'flexible',\n",
       " 'prominent',\n",
       " 'agile',\n",
       " 'tidy',\n",
       " 'invaluable',\n",
       " 'sleek',\n",
       " 'hallmark',\n",
       " 'refund',\n",
       " 'noble',\n",
       " 'unity',\n",
       " 'paradise',\n",
       " 'heartfelt',\n",
       " 'pros',\n",
       " 'restructuring',\n",
       " 'encouragement',\n",
       " 'humor',\n",
       " 'unrestricted',\n",
       " 'unreal',\n",
       " 'stabilize',\n",
       " 'quaint',\n",
       " 'appeal',\n",
       " 'softer',\n",
       " 'easiest',\n",
       " 'monumental',\n",
       " 'satisfying',\n",
       " 'distinctive',\n",
       " 'mighty',\n",
       " 'elite',\n",
       " 'robust',\n",
       " 'engaging',\n",
       " 'satisfactory',\n",
       " 'prosperous',\n",
       " 'adaptive',\n",
       " 'thrill',\n",
       " 'treasure',\n",
       " 'golden',\n",
       " 'splendid',\n",
       " 'enough',\n",
       " 'clever',\n",
       " 'substantive',\n",
       " 'freed',\n",
       " 'compact',\n",
       " 'desirable',\n",
       " 'proactive',\n",
       " 'helping',\n",
       " 'enticing',\n",
       " 'merry',\n",
       " 'affirmative',\n",
       " 'famed',\n",
       " 'warmer',\n",
       " 'gold',\n",
       " 'sublime',\n",
       " 'heaven',\n",
       " 'reform',\n",
       " 'innovation',\n",
       " 'wow',\n",
       " 'nifty',\n",
       " 'favored',\n",
       " 'excel',\n",
       " 'prompt',\n",
       " 'respect',\n",
       " 'breakthrough',\n",
       " 'breeze',\n",
       " 'reformed',\n",
       " 'attraction',\n",
       " 'enjoy',\n",
       " 'protective',\n",
       " 'poignant',\n",
       " 'holy',\n",
       " 'happiness',\n",
       " 'incredibly',\n",
       " 'thoughtful',\n",
       " 'helped',\n",
       " 'fruitful',\n",
       " 'imaginative',\n",
       " 'impressed',\n",
       " 'credible',\n",
       " 'praise',\n",
       " 'simplest',\n",
       " 'pride',\n",
       " 'award',\n",
       " 'guarantee',\n",
       " 'priceless',\n",
       " 'satisfied',\n",
       " 'generous',\n",
       " 'valiant',\n",
       " 'willing',\n",
       " 'skilled',\n",
       " 'fans',\n",
       " 'savings',\n",
       " 'comforting',\n",
       " 'awards',\n",
       " 'renewed',\n",
       " 'smiling',\n",
       " 'honorable',\n",
       " 'adequate',\n",
       " 'beneficial',\n",
       " 'standout',\n",
       " 'winners',\n",
       " 'orderly',\n",
       " 'upscale',\n",
       " 'enjoyed',\n",
       " 'prefer',\n",
       " 'ethical',\n",
       " 'led',\n",
       " 'illustrious',\n",
       " 'backbone',\n",
       " 'cohesive',\n",
       " 'tougher',\n",
       " 'eager',\n",
       " 'flashy',\n",
       " 'influential',\n",
       " 'stylish',\n",
       " 'lucid',\n",
       " 'luck',\n",
       " 'brighter',\n",
       " 'attentive',\n",
       " 'fulfillment',\n",
       " 'adjustable',\n",
       " 'enjoying',\n",
       " 'commitment',\n",
       " 'masters',\n",
       " 'unbiased',\n",
       " 'wonder',\n",
       " 'prudent',\n",
       " 'defeat',\n",
       " 'audible',\n",
       " 'believable',\n",
       " 'permissible',\n",
       " 'dominate',\n",
       " 'cheer',\n",
       " 'prosperity',\n",
       " 'encourage',\n",
       " 'dominated',\n",
       " 'assurance',\n",
       " 'stronger',\n",
       " 'compliant',\n",
       " 'clarity',\n",
       " 'admire',\n",
       " 'liked',\n",
       " 'recommendation',\n",
       " 'gracious',\n",
       " 'cheapest',\n",
       " 'admiration',\n",
       " 'enhancement',\n",
       " 'glory',\n",
       " 'handy',\n",
       " 'trusted',\n",
       " 'bargain',\n",
       " 'faith',\n",
       " 'gained',\n",
       " 'promise',\n",
       " 'lucrative',\n",
       " 'boom',\n",
       " 'acclaimed',\n",
       " 'classy',\n",
       " 'novelty',\n",
       " 'aspiration',\n",
       " 'grateful',\n",
       " 'solidarity',\n",
       " 'renowned',\n",
       " 'winner',\n",
       " 'privilege',\n",
       " 'handsome',\n",
       " 'sincere',\n",
       " 'avid',\n",
       " 'capability',\n",
       " 'earnest',\n",
       " 'inspire',\n",
       " 'advantageous',\n",
       " 'humane',\n",
       " 'charitable',\n",
       " 'vibrant',\n",
       " 'fearless',\n",
       " 'reputable',\n",
       " 'reclaim',\n",
       " 'supporter',\n",
       " 'agility',\n",
       " 'celebrate',\n",
       " 'endorsement',\n",
       " 'fairly',\n",
       " 'enchanted',\n",
       " 'richer',\n",
       " 'cherished',\n",
       " 'admirable',\n",
       " 'prestigious',\n",
       " 'smarter',\n",
       " 'compassionate',\n",
       " 'sharper',\n",
       " 'pleased',\n",
       " 'sweeping',\n",
       " 'durable',\n",
       " 'affirmation',\n",
       " 'lavish',\n",
       " 'readable',\n",
       " 'pleasing',\n",
       " 'smoothly',\n",
       " 'achievement',\n",
       " 'supports',\n",
       " 'boost',\n",
       " 'endorsed',\n",
       " 'blessing',\n",
       " 'courage',\n",
       " 'idol',\n",
       " 'sufficient',\n",
       " 'seasoned',\n",
       " 'achievable',\n",
       " 'delighted',\n",
       " 'genius',\n",
       " 'talent',\n",
       " 'fond',\n",
       " 'reasoned',\n",
       " 'clearly',\n",
       " 'glowing',\n",
       " 'liberation',\n",
       " 'agreeable',\n",
       " 'successfully',\n",
       " 'profound',\n",
       " 'advocate',\n",
       " 'eased',\n",
       " 'cozy',\n",
       " 'rapt',\n",
       " 'greatest',\n",
       " 'thinner',\n",
       " 'amazingly',\n",
       " 'congratulations',\n",
       " 'sensation',\n",
       " 'charm',\n",
       " 'darling',\n",
       " 'reputation',\n",
       " 'regard',\n",
       " 'premier',\n",
       " 'intimacy',\n",
       " 'champion',\n",
       " 'harmony',\n",
       " 'gain',\n",
       " 'booming',\n",
       " 'gem',\n",
       " 'immense',\n",
       " 'leverage',\n",
       " 'flattering',\n",
       " 'virtuous',\n",
       " 'triumph',\n",
       " 'inventive',\n",
       " 'cure',\n",
       " 'awarded',\n",
       " 'goodwill',\n",
       " 'smile',\n",
       " 'improvements',\n",
       " 'accomplishment',\n",
       " 'morality',\n",
       " 'dazzling',\n",
       " 'shiny',\n",
       " 'commend',\n",
       " 'amazed',\n",
       " 'keen',\n",
       " 'authoritative',\n",
       " 'sturdy',\n",
       " 'beautifully',\n",
       " 'benefits',\n",
       " 'thank',\n",
       " 'favor',\n",
       " 'trust',\n",
       " 'simplify',\n",
       " 'recommend',\n",
       " 'refresh',\n",
       " 'approve',\n",
       " 'receptive',\n",
       " 'facilitate',\n",
       " 'resilient',\n",
       " 'complement',\n",
       " 'plush',\n",
       " 'skill',\n",
       " 'merit',\n",
       " 'catchy',\n",
       " 'fluent',\n",
       " 'protection',\n",
       " 'upheld',\n",
       " 'enlightenment',\n",
       " 'enjoyment',\n",
       " 'diligent',\n",
       " 'likes',\n",
       " 'truthful',\n",
       " 'lush',\n",
       " 'enthusiasm',\n",
       " 'compliment',\n",
       " 'compassion',\n",
       " 'succeeded',\n",
       " 'loyal',\n",
       " 'leads',\n",
       " 'youthful',\n",
       " 'easing',\n",
       " 'delight',\n",
       " 'mastery',\n",
       " 'trustworthy',\n",
       " 'glad',\n",
       " 'marvel',\n",
       " 'superiority',\n",
       " 'awe',\n",
       " 'humour',\n",
       " 'goodness',\n",
       " 'appreciate',\n",
       " 'wins',\n",
       " 'positives',\n",
       " 'fav',\n",
       " 'affection',\n",
       " 'chic',\n",
       " 'gains',\n",
       " 'wisdom',\n",
       " 'perfectly',\n",
       " 'trendy',\n",
       " 'thriving',\n",
       " 'plentiful',\n",
       " 'promoter',\n",
       " 'impartial',\n",
       " 'trumpet',\n",
       " 'qualify',\n",
       " 'patience',\n",
       " 'loves',\n",
       " 'gratitude',\n",
       " 'knowledgeable',\n",
       " 'optimism',\n",
       " 'hottest',\n",
       " 'formidable',\n",
       " 'advocates',\n",
       " 'happily',\n",
       " 'proves',\n",
       " 'prolific',\n",
       " 'ample',\n",
       " 'remedy',\n",
       " 'talented',\n",
       " 'savvy',\n",
       " 'envy',\n",
       " 'reasonably',\n",
       " 'sustainability',\n",
       " 'masterpiece',\n",
       " 'principled',\n",
       " 'gaining',\n",
       " 'fidelity',\n",
       " 'coolest',\n",
       " 'honoring',\n",
       " 'stunned',\n",
       " 'safely',\n",
       " 'lover',\n",
       " 'lawful',\n",
       " 'successes',\n",
       " 'witty',\n",
       " 'unmatched',\n",
       " 'improves',\n",
       " 'properly',\n",
       " 'ingenious',\n",
       " 'promises',\n",
       " 'salute',\n",
       " 'stimulating',\n",
       " 'empowerment',\n",
       " 'enhance',\n",
       " 'unparalleled',\n",
       " 'articulate',\n",
       " 'equitable',\n",
       " 'hug',\n",
       " 'empathy',\n",
       " 'abundant',\n",
       " 'advantages',\n",
       " 'refine',\n",
       " 'enrichment',\n",
       " 'sensations',\n",
       " 'protect',\n",
       " 'contribution',\n",
       " 'visionary',\n",
       " 'praising',\n",
       " 'defender',\n",
       " 'bliss',\n",
       " 'renaissance',\n",
       " 'ecstasy',\n",
       " 'bless',\n",
       " 'devout',\n",
       " 'unconditional',\n",
       " 'fortunately',\n",
       " 'effectiveness',\n",
       " 'revive',\n",
       " 'eminent',\n",
       " 'flexibility',\n",
       " 'perfection',\n",
       " 'intrigue',\n",
       " 'gratification',\n",
       " 'advocated',\n",
       " 'supremacy',\n",
       " 'enrich',\n",
       " 'affirm',\n",
       " 'fertile',\n",
       " 'affluent',\n",
       " 'amuse',\n",
       " 'remarkably',\n",
       " 'distinction',\n",
       " 'congratulate',\n",
       " 'prefers',\n",
       " 'exalted',\n",
       " 'maturity',\n",
       " 'fashionable',\n",
       " 'elegance',\n",
       " 'astonished',\n",
       " 'everlasting',\n",
       " 'peach',\n",
       " 'exceptionally',\n",
       " 'trump',\n",
       " 'poised',\n",
       " 'succeed',\n",
       " 'notably',\n",
       " 'reliably',\n",
       " 'revelation',\n",
       " 'finer',\n",
       " 'tremendously',\n",
       " 'proficient',\n",
       " 'refinement',\n",
       " 'glow',\n",
       " 'fairness',\n",
       " 'wealthy',\n",
       " 'irresistible',\n",
       " 'versatile',\n",
       " 'achievements',\n",
       " 'nicely',\n",
       " 'dignity',\n",
       " 'fame',\n",
       " 'radiant',\n",
       " 'flourishing',\n",
       " 'destiny',\n",
       " 'exceeded',\n",
       " 'ardent',\n",
       " 'accomplish',\n",
       " 'ease',\n",
       " 'kindly',\n",
       " 'correctly',\n",
       " 'aspirations',\n",
       " 'consistently',\n",
       " 'humility',\n",
       " 'loyalty',\n",
       " 'rapport',\n",
       " 'sparkling',\n",
       " 'defeating',\n",
       " 'comfortably',\n",
       " 'revel',\n",
       " 'deft',\n",
       " 'fortune',\n",
       " 'conscientious',\n",
       " 'glitter',\n",
       " 'redeem',\n",
       " 'reforming',\n",
       " 'grin',\n",
       " 'recommendations',\n",
       " 'overtake',\n",
       " 'patriot',\n",
       " 'diligence',\n",
       " 'efficiently',\n",
       " 'smiles',\n",
       " 'thrilled',\n",
       " 'majesty',\n",
       " 'savior',\n",
       " 'cornerstone',\n",
       " 'supreme',\n",
       " 'affinity',\n",
       " 'assure',\n",
       " 'paramount',\n",
       " 'remission',\n",
       " 'bloom',\n",
       " 'accomplishments',\n",
       " 'heal',\n",
       " 'reassure',\n",
       " 'interests',\n",
       " 'liking',\n",
       " 'invincible',\n",
       " 'precisely',\n",
       " 'instantly',\n",
       " 'charismatic',\n",
       " 'rejoice',\n",
       " 'clears',\n",
       " 'abundance',\n",
       " 'nurturing',\n",
       " 'illuminate',\n",
       " 'gifted',\n",
       " 'prize',\n",
       " 'famously',\n",
       " 'tops',\n",
       " 'champ',\n",
       " 'trusting',\n",
       " 'flourish',\n",
       " 'traction',\n",
       " 'beneficiary',\n",
       " 'applaud',\n",
       " 'impress',\n",
       " 'brightest',\n",
       " 'finest',\n",
       " 'kindness',\n",
       " 'defeats',\n",
       " 'openly',\n",
       " 'accurately',\n",
       " 'rightful',\n",
       " 'benevolent',\n",
       " 'strongest',\n",
       " 'ideally',\n",
       " 'entertain',\n",
       " 'stainless',\n",
       " 'enhances',\n",
       " 'dominates',\n",
       " 'steadfast',\n",
       " 'afford',\n",
       " 'sufficiently',\n",
       " 'warmth',\n",
       " 'indebted',\n",
       " 'prosper',\n",
       " 'vigilant',\n",
       " 'wonderfully',\n",
       " 'laud',\n",
       " 'favour',\n",
       " 'woo',\n",
       " 'fascination',\n",
       " 'enchant',\n",
       " 'extraordinarily',\n",
       " 'succeeds',\n",
       " 'enjoys',\n",
       " 'empower',\n",
       " 'aver',\n",
       " 'reforms',\n",
       " 'toughest',\n",
       " 'saint',\n",
       " 'honesty',\n",
       " 'reconcile',\n",
       " 'effectively',\n",
       " 'pardon',\n",
       " 'eagerly',\n",
       " 'satisfy',\n",
       " 'faithfully',\n",
       " 'decency',\n",
       " 'bolster',\n",
       " 'aspire',\n",
       " 'thrive',\n",
       " 'liberate',\n",
       " 'prominence',\n",
       " 'elevate',\n",
       " 'deserving',\n",
       " 'greatness',\n",
       " 'renown',\n",
       " 'bravery',\n",
       " 'preferring',\n",
       " 'feat',\n",
       " 'promptly',\n",
       " 'excellence',\n",
       " 'miracles',\n",
       " 'strikingly',\n",
       " 'assurances',\n",
       " 'endorse',\n",
       " 'gems',\n",
       " 'reverence',\n",
       " 'endorsing',\n",
       " 'covenant',\n",
       " 'peacefully',\n",
       " 'suffice',\n",
       " 'unequivocally',\n",
       " 'shine',\n",
       " 'vigilance',\n",
       " 'conveniently',\n",
       " 'stimulate',\n",
       " 'adore',\n",
       " 'bonuses',\n",
       " 'striving',\n",
       " 'freedoms',\n",
       " 'wellbeing',\n",
       " 'ingenuity',\n",
       " 'effortlessly',\n",
       " 'positively',\n",
       " 'heroine',\n",
       " 'exceeding',\n",
       " 'wonders',\n",
       " 'obsession',\n",
       " 'brilliance',\n",
       " 'acclaim',\n",
       " 'readily',\n",
       " 'openness',\n",
       " 'pleasantly',\n",
       " 'cherish',\n",
       " 'abound',\n",
       " 'enthusiastically',\n",
       " 'brilliantly',\n",
       " 'staunch',\n",
       " 'virtue',\n",
       " 'willingly',\n",
       " 'surpass',\n",
       " 'preferably',\n",
       " 'versatility',\n",
       " 'exceeds',\n",
       " 'smartest',\n",
       " 'neatly',\n",
       " 'finely',\n",
       " 'sincerely',\n",
       " 'generosity',\n",
       " 'tempt',\n",
       " 'charisma',\n",
       " 'meticulously',\n",
       " 'respectfully',\n",
       " 'exceed',\n",
       " 'prowess',\n",
       " 'evenly',\n",
       " ...]"
      ]
     },
     "execution_count": 615,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.suggest('This is {a:bert} flight.', verbose=True, candidates=['Ġ' + x for x in pos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is a a+ thing.',\n",
       " 'This is a abound thing.',\n",
       " 'This is a abounds thing.',\n",
       " 'This is a abundance thing.',\n",
       " 'This is a abundant thing.',\n",
       " 'This is a accessable thing.',\n",
       " 'This is a accessible thing.',\n",
       " 'This is a acclaim thing.',\n",
       " 'This is a acclaimed thing.',\n",
       " 'This is a acclamation thing.',\n",
       " 'This is a accolade thing.',\n",
       " 'This is a accolades thing.',\n",
       " 'This is a accommodative thing.',\n",
       " 'This is a accomodative thing.',\n",
       " 'This is a accomplish thing.',\n",
       " 'This is a accomplished thing.',\n",
       " 'This is a accomplishment thing.',\n",
       " 'This is a accomplishments thing.',\n",
       " 'This is a accurate thing.',\n",
       " 'This is a accurately thing.',\n",
       " 'This is a achievable thing.',\n",
       " 'This is a achievement thing.',\n",
       " 'This is a achievements thing.',\n",
       " 'This is a achievible thing.',\n",
       " 'This is a acumen thing.',\n",
       " 'This is a adaptable thing.',\n",
       " 'This is a adaptive thing.',\n",
       " 'This is a adequate thing.',\n",
       " 'This is a adjustable thing.',\n",
       " 'This is a admirable thing.',\n",
       " 'This is a admirably thing.',\n",
       " 'This is a admiration thing.',\n",
       " 'This is a admire thing.',\n",
       " 'This is a admirer thing.',\n",
       " 'This is a admiring thing.',\n",
       " 'This is a admiringly thing.',\n",
       " 'This is a adorable thing.',\n",
       " 'This is a adore thing.',\n",
       " 'This is a adored thing.',\n",
       " 'This is a adorer thing.',\n",
       " 'This is a adoring thing.',\n",
       " 'This is a adoringly thing.',\n",
       " 'This is a adroit thing.',\n",
       " 'This is a adroitly thing.',\n",
       " 'This is a adulate thing.',\n",
       " 'This is a adulation thing.',\n",
       " 'This is a adulatory thing.',\n",
       " 'This is a advanced thing.',\n",
       " 'This is a advantage thing.',\n",
       " 'This is a advantageous thing.',\n",
       " 'This is a advantageously thing.',\n",
       " 'This is a advantages thing.',\n",
       " 'This is a adventuresome thing.',\n",
       " 'This is a adventurous thing.',\n",
       " 'This is a advocate thing.',\n",
       " 'This is a advocated thing.',\n",
       " 'This is a advocates thing.',\n",
       " 'This is a affability thing.',\n",
       " 'This is a affable thing.',\n",
       " 'This is a affably thing.',\n",
       " 'This is a affectation thing.',\n",
       " 'This is a affection thing.',\n",
       " 'This is a affectionate thing.',\n",
       " 'This is a affinity thing.',\n",
       " 'This is a affirm thing.',\n",
       " 'This is a affirmation thing.',\n",
       " 'This is a affirmative thing.',\n",
       " 'This is a affluence thing.',\n",
       " 'This is a affluent thing.',\n",
       " 'This is a afford thing.',\n",
       " 'This is a affordable thing.',\n",
       " 'This is a affordably thing.',\n",
       " 'This is a afordable thing.',\n",
       " 'This is a agile thing.',\n",
       " 'This is a agilely thing.',\n",
       " 'This is a agility thing.',\n",
       " 'This is a agreeable thing.',\n",
       " 'This is a agreeableness thing.',\n",
       " 'This is a agreeably thing.',\n",
       " 'This is a all-around thing.',\n",
       " 'This is a alluring thing.',\n",
       " 'This is a alluringly thing.',\n",
       " 'This is a altruistic thing.',\n",
       " 'This is a altruistically thing.',\n",
       " 'This is a amaze thing.',\n",
       " 'This is a amazed thing.',\n",
       " 'This is a amazement thing.',\n",
       " 'This is a amazes thing.',\n",
       " 'This is a amazing thing.',\n",
       " 'This is a amazingly thing.',\n",
       " 'This is a ambitious thing.',\n",
       " 'This is a ambitiously thing.',\n",
       " 'This is a ameliorate thing.',\n",
       " 'This is a amenable thing.',\n",
       " 'This is a amenity thing.',\n",
       " 'This is a amiability thing.',\n",
       " 'This is a amiabily thing.',\n",
       " 'This is a amiable thing.',\n",
       " 'This is a amicability thing.',\n",
       " 'This is a amicable thing.',\n",
       " 'This is a amicably thing.',\n",
       " 'This is a amity thing.',\n",
       " 'This is a ample thing.',\n",
       " 'This is a amply thing.',\n",
       " 'This is a amuse thing.',\n",
       " 'This is a amusing thing.',\n",
       " 'This is a amusingly thing.',\n",
       " 'This is a angel thing.',\n",
       " 'This is a angelic thing.',\n",
       " 'This is a apotheosis thing.',\n",
       " 'This is a appeal thing.',\n",
       " 'This is a appealing thing.',\n",
       " 'This is a applaud thing.',\n",
       " 'This is a appreciable thing.',\n",
       " 'This is a appreciate thing.',\n",
       " 'This is a appreciated thing.',\n",
       " 'This is a appreciates thing.',\n",
       " 'This is a appreciative thing.',\n",
       " 'This is a appreciatively thing.',\n",
       " 'This is a appropriate thing.',\n",
       " 'This is a approval thing.',\n",
       " 'This is a approve thing.',\n",
       " 'This is a ardent thing.',\n",
       " 'This is a ardently thing.',\n",
       " 'This is a ardor thing.',\n",
       " 'This is a articulate thing.',\n",
       " 'This is a aspiration thing.',\n",
       " 'This is a aspirations thing.',\n",
       " 'This is a aspire thing.',\n",
       " 'This is a assurance thing.',\n",
       " 'This is a assurances thing.',\n",
       " 'This is a assure thing.',\n",
       " 'This is a assuredly thing.',\n",
       " 'This is a assuring thing.',\n",
       " 'This is a astonish thing.',\n",
       " 'This is a astonished thing.',\n",
       " 'This is a astonishing thing.',\n",
       " 'This is a astonishingly thing.',\n",
       " 'This is a astonishment thing.',\n",
       " 'This is a astound thing.',\n",
       " 'This is a astounded thing.',\n",
       " 'This is a astounding thing.',\n",
       " 'This is a astoundingly thing.',\n",
       " 'This is a astutely thing.',\n",
       " 'This is a attentive thing.',\n",
       " 'This is a attraction thing.',\n",
       " 'This is a attractive thing.',\n",
       " 'This is a attractively thing.',\n",
       " 'This is a attune thing.',\n",
       " 'This is a audible thing.',\n",
       " 'This is a audibly thing.',\n",
       " 'This is a auspicious thing.',\n",
       " 'This is a authentic thing.',\n",
       " 'This is a authoritative thing.',\n",
       " 'This is a autonomous thing.',\n",
       " 'This is a available thing.',\n",
       " 'This is a aver thing.',\n",
       " 'This is a avid thing.',\n",
       " 'This is a avidly thing.',\n",
       " 'This is a award thing.',\n",
       " 'This is a awarded thing.',\n",
       " 'This is a awards thing.',\n",
       " 'This is a awe thing.',\n",
       " 'This is a awed thing.',\n",
       " 'This is a awesome thing.',\n",
       " 'This is a awesomely thing.',\n",
       " 'This is a awesomeness thing.',\n",
       " 'This is a awestruck thing.',\n",
       " 'This is a awsome thing.',\n",
       " 'This is a backbone thing.',\n",
       " 'This is a balanced thing.',\n",
       " 'This is a bargain thing.',\n",
       " 'This is a beauteous thing.',\n",
       " 'This is a beautiful thing.',\n",
       " 'This is a beautifullly thing.',\n",
       " 'This is a beautifully thing.',\n",
       " 'This is a beautify thing.',\n",
       " 'This is a beauty thing.',\n",
       " 'This is a beckon thing.',\n",
       " 'This is a beckoned thing.',\n",
       " 'This is a beckoning thing.',\n",
       " 'This is a beckons thing.',\n",
       " 'This is a believable thing.',\n",
       " 'This is a believeable thing.',\n",
       " 'This is a beloved thing.',\n",
       " 'This is a benefactor thing.',\n",
       " 'This is a beneficent thing.',\n",
       " 'This is a beneficial thing.',\n",
       " 'This is a beneficially thing.',\n",
       " 'This is a beneficiary thing.',\n",
       " 'This is a benefit thing.',\n",
       " 'This is a benefits thing.',\n",
       " 'This is a benevolence thing.',\n",
       " 'This is a benevolent thing.',\n",
       " 'This is a benifits thing.',\n",
       " 'This is a best thing.',\n",
       " 'This is a best-known thing.',\n",
       " 'This is a best-performing thing.',\n",
       " 'This is a best-selling thing.',\n",
       " 'This is a better thing.',\n",
       " 'This is a better-known thing.',\n",
       " 'This is a better-than-expected thing.',\n",
       " 'This is a beutifully thing.',\n",
       " 'This is a blameless thing.',\n",
       " 'This is a bless thing.',\n",
       " 'This is a blessing thing.',\n",
       " 'This is a bliss thing.',\n",
       " 'This is a blissful thing.',\n",
       " 'This is a blissfully thing.',\n",
       " 'This is a blithe thing.',\n",
       " 'This is a blockbuster thing.',\n",
       " 'This is a bloom thing.',\n",
       " 'This is a blossom thing.',\n",
       " 'This is a bolster thing.',\n",
       " 'This is a bonny thing.',\n",
       " 'This is a bonus thing.',\n",
       " 'This is a bonuses thing.',\n",
       " 'This is a boom thing.',\n",
       " 'This is a booming thing.',\n",
       " 'This is a boost thing.',\n",
       " 'This is a boundless thing.',\n",
       " 'This is a bountiful thing.',\n",
       " 'This is a brainiest thing.',\n",
       " 'This is a brainy thing.',\n",
       " 'This is a brand-new thing.',\n",
       " 'This is a brave thing.',\n",
       " 'This is a bravery thing.',\n",
       " 'This is a bravo thing.',\n",
       " 'This is a breakthrough thing.',\n",
       " 'This is a breakthroughs thing.',\n",
       " 'This is a breathlessness thing.',\n",
       " 'This is a breathtaking thing.',\n",
       " 'This is a breathtakingly thing.',\n",
       " 'This is a breeze thing.',\n",
       " 'This is a bright thing.',\n",
       " 'This is a brighten thing.',\n",
       " 'This is a brighter thing.',\n",
       " 'This is a brightest thing.',\n",
       " 'This is a brilliance thing.',\n",
       " 'This is a brilliances thing.',\n",
       " 'This is a brilliant thing.',\n",
       " 'This is a brilliantly thing.',\n",
       " 'This is a brisk thing.',\n",
       " 'This is a brotherly thing.',\n",
       " 'This is a bullish thing.',\n",
       " 'This is a buoyant thing.',\n",
       " 'This is a cajole thing.',\n",
       " 'This is a calm thing.',\n",
       " 'This is a calming thing.',\n",
       " 'This is a calmness thing.',\n",
       " 'This is a capability thing.',\n",
       " 'This is a capable thing.',\n",
       " 'This is a capably thing.',\n",
       " 'This is a captivate thing.',\n",
       " 'This is a captivating thing.',\n",
       " 'This is a carefree thing.',\n",
       " 'This is a cashback thing.',\n",
       " 'This is a cashbacks thing.',\n",
       " 'This is a catchy thing.',\n",
       " 'This is a celebrate thing.',\n",
       " 'This is a celebrated thing.',\n",
       " 'This is a celebration thing.',\n",
       " 'This is a celebratory thing.',\n",
       " 'This is a champ thing.',\n",
       " 'This is a champion thing.',\n",
       " 'This is a charisma thing.',\n",
       " 'This is a charismatic thing.',\n",
       " 'This is a charitable thing.',\n",
       " 'This is a charm thing.',\n",
       " 'This is a charming thing.',\n",
       " 'This is a charmingly thing.',\n",
       " 'This is a chaste thing.',\n",
       " 'This is a cheaper thing.',\n",
       " 'This is a cheapest thing.',\n",
       " 'This is a cheer thing.',\n",
       " 'This is a cheerful thing.',\n",
       " 'This is a cheery thing.',\n",
       " 'This is a cherish thing.',\n",
       " 'This is a cherished thing.',\n",
       " 'This is a cherub thing.',\n",
       " 'This is a chic thing.',\n",
       " 'This is a chivalrous thing.',\n",
       " 'This is a chivalry thing.',\n",
       " 'This is a civility thing.',\n",
       " 'This is a civilize thing.',\n",
       " 'This is a clarity thing.',\n",
       " 'This is a classic thing.',\n",
       " 'This is a classy thing.',\n",
       " 'This is a clean thing.',\n",
       " 'This is a cleaner thing.',\n",
       " 'This is a cleanest thing.',\n",
       " 'This is a cleanliness thing.',\n",
       " 'This is a cleanly thing.',\n",
       " 'This is a clear thing.',\n",
       " 'This is a clear-cut thing.',\n",
       " 'This is a cleared thing.',\n",
       " 'This is a clearer thing.',\n",
       " 'This is a clearly thing.',\n",
       " 'This is a clears thing.',\n",
       " 'This is a clever thing.',\n",
       " 'This is a cleverly thing.',\n",
       " 'This is a cohere thing.',\n",
       " 'This is a coherence thing.',\n",
       " 'This is a coherent thing.',\n",
       " 'This is a cohesive thing.',\n",
       " 'This is a colorful thing.',\n",
       " 'This is a comely thing.',\n",
       " 'This is a comfort thing.',\n",
       " 'This is a comfortable thing.',\n",
       " 'This is a comfortably thing.',\n",
       " 'This is a comforting thing.',\n",
       " 'This is a comfy thing.',\n",
       " 'This is a commend thing.',\n",
       " 'This is a commendable thing.',\n",
       " 'This is a commendably thing.',\n",
       " 'This is a commitment thing.',\n",
       " 'This is a commodious thing.',\n",
       " 'This is a compact thing.',\n",
       " 'This is a compactly thing.',\n",
       " 'This is a compassion thing.',\n",
       " 'This is a compassionate thing.',\n",
       " 'This is a compatible thing.',\n",
       " 'This is a competitive thing.',\n",
       " 'This is a complement thing.',\n",
       " 'This is a complementary thing.',\n",
       " 'This is a complemented thing.',\n",
       " 'This is a complements thing.',\n",
       " 'This is a compliant thing.',\n",
       " 'This is a compliment thing.',\n",
       " 'This is a complimentary thing.',\n",
       " 'This is a comprehensive thing.',\n",
       " 'This is a conciliate thing.',\n",
       " 'This is a conciliatory thing.',\n",
       " 'This is a concise thing.',\n",
       " 'This is a confidence thing.',\n",
       " 'This is a confident thing.',\n",
       " 'This is a congenial thing.',\n",
       " 'This is a congratulate thing.',\n",
       " 'This is a congratulation thing.',\n",
       " 'This is a congratulations thing.',\n",
       " 'This is a congratulatory thing.',\n",
       " 'This is a conscientious thing.',\n",
       " 'This is a considerate thing.',\n",
       " 'This is a consistent thing.',\n",
       " 'This is a consistently thing.',\n",
       " 'This is a constructive thing.',\n",
       " 'This is a consummate thing.',\n",
       " 'This is a contentment thing.',\n",
       " 'This is a continuity thing.',\n",
       " 'This is a contrasty thing.',\n",
       " 'This is a contribution thing.',\n",
       " 'This is a convenience thing.',\n",
       " 'This is a convenient thing.',\n",
       " 'This is a conveniently thing.',\n",
       " 'This is a convience thing.',\n",
       " 'This is a convienient thing.',\n",
       " 'This is a convient thing.',\n",
       " 'This is a convincing thing.',\n",
       " 'This is a convincingly thing.',\n",
       " 'This is a cool thing.',\n",
       " 'This is a coolest thing.',\n",
       " 'This is a cooperative thing.',\n",
       " 'This is a cooperatively thing.',\n",
       " 'This is a cornerstone thing.',\n",
       " 'This is a correct thing.',\n",
       " 'This is a correctly thing.',\n",
       " 'This is a cost-effective thing.',\n",
       " 'This is a cost-saving thing.',\n",
       " 'This is a counter-attack thing.',\n",
       " 'This is a counter-attacks thing.',\n",
       " 'This is a courage thing.',\n",
       " 'This is a courageous thing.',\n",
       " 'This is a courageously thing.',\n",
       " 'This is a courageousness thing.',\n",
       " 'This is a courteous thing.',\n",
       " 'This is a courtly thing.',\n",
       " 'This is a covenant thing.',\n",
       " 'This is a cozy thing.',\n",
       " 'This is a creative thing.',\n",
       " 'This is a credence thing.',\n",
       " 'This is a credible thing.',\n",
       " 'This is a crisp thing.',\n",
       " 'This is a crisper thing.',\n",
       " 'This is a cure thing.',\n",
       " 'This is a cure-all thing.',\n",
       " 'This is a cushy thing.',\n",
       " 'This is a cute thing.',\n",
       " 'This is a cuteness thing.',\n",
       " 'This is a danke thing.',\n",
       " 'This is a danken thing.',\n",
       " 'This is a daring thing.',\n",
       " 'This is a daringly thing.',\n",
       " 'This is a darling thing.',\n",
       " 'This is a dashing thing.',\n",
       " 'This is a dauntless thing.',\n",
       " 'This is a dawn thing.',\n",
       " 'This is a dazzle thing.',\n",
       " 'This is a dazzled thing.',\n",
       " 'This is a dazzling thing.',\n",
       " 'This is a dead-cheap thing.',\n",
       " 'This is a dead-on thing.',\n",
       " 'This is a decency thing.',\n",
       " 'This is a decent thing.',\n",
       " 'This is a decisive thing.',\n",
       " 'This is a decisiveness thing.',\n",
       " 'This is a dedicated thing.',\n",
       " 'This is a defeat thing.',\n",
       " 'This is a defeated thing.',\n",
       " 'This is a defeating thing.',\n",
       " 'This is a defeats thing.',\n",
       " 'This is a defender thing.',\n",
       " 'This is a deference thing.',\n",
       " 'This is a deft thing.',\n",
       " 'This is a deginified thing.',\n",
       " 'This is a delectable thing.',\n",
       " 'This is a delicacy thing.',\n",
       " 'This is a delicate thing.',\n",
       " 'This is a delicious thing.',\n",
       " 'This is a delight thing.',\n",
       " 'This is a delighted thing.',\n",
       " 'This is a delightful thing.',\n",
       " 'This is a delightfully thing.',\n",
       " 'This is a delightfulness thing.',\n",
       " 'This is a dependable thing.',\n",
       " 'This is a dependably thing.',\n",
       " 'This is a deservedly thing.',\n",
       " 'This is a deserving thing.',\n",
       " 'This is a desirable thing.',\n",
       " 'This is a desiring thing.',\n",
       " 'This is a desirous thing.',\n",
       " 'This is a destiny thing.',\n",
       " 'This is a detachable thing.',\n",
       " 'This is a devout thing.',\n",
       " 'This is a dexterous thing.',\n",
       " 'This is a dexterously thing.',\n",
       " 'This is a dextrous thing.',\n",
       " 'This is a dignified thing.',\n",
       " 'This is a dignify thing.',\n",
       " 'This is a dignity thing.',\n",
       " 'This is a diligence thing.',\n",
       " 'This is a diligent thing.',\n",
       " 'This is a diligently thing.',\n",
       " 'This is a diplomatic thing.',\n",
       " 'This is a dirt-cheap thing.',\n",
       " 'This is a distinction thing.',\n",
       " 'This is a distinctive thing.',\n",
       " 'This is a distinguished thing.',\n",
       " 'This is a diversified thing.',\n",
       " 'This is a divine thing.',\n",
       " 'This is a divinely thing.',\n",
       " 'This is a dominate thing.',\n",
       " 'This is a dominated thing.',\n",
       " 'This is a dominates thing.',\n",
       " 'This is a dote thing.',\n",
       " 'This is a dotingly thing.',\n",
       " 'This is a doubtless thing.',\n",
       " 'This is a dreamland thing.',\n",
       " 'This is a dumbfounded thing.',\n",
       " 'This is a dumbfounding thing.',\n",
       " 'This is a dummy-proof thing.',\n",
       " 'This is a durable thing.',\n",
       " 'This is a dynamic thing.',\n",
       " 'This is a eager thing.',\n",
       " 'This is a eagerly thing.',\n",
       " 'This is a eagerness thing.',\n",
       " 'This is a earnest thing.',\n",
       " 'This is a earnestly thing.',\n",
       " 'This is a earnestness thing.',\n",
       " 'This is a ease thing.',\n",
       " 'This is a eased thing.',\n",
       " 'This is a eases thing.',\n",
       " 'This is a easier thing.',\n",
       " 'This is a easiest thing.',\n",
       " 'This is a easiness thing.',\n",
       " 'This is a easing thing.',\n",
       " 'This is a easy thing.',\n",
       " 'This is a easy-to-use thing.',\n",
       " 'This is a easygoing thing.',\n",
       " 'This is a ebullience thing.',\n",
       " 'This is a ebullient thing.',\n",
       " 'This is a ebulliently thing.',\n",
       " 'This is a ecenomical thing.',\n",
       " 'This is a economical thing.',\n",
       " 'This is a ecstasies thing.',\n",
       " 'This is a ecstasy thing.',\n",
       " 'This is a ecstatic thing.',\n",
       " 'This is a ecstatically thing.',\n",
       " 'This is a edify thing.',\n",
       " 'This is a educated thing.',\n",
       " 'This is a effective thing.',\n",
       " 'This is a effectively thing.',\n",
       " 'This is a effectiveness thing.',\n",
       " 'This is a effectual thing.',\n",
       " 'This is a efficacious thing.',\n",
       " 'This is a efficient thing.',\n",
       " 'This is a efficiently thing.',\n",
       " 'This is a effortless thing.',\n",
       " 'This is a effortlessly thing.',\n",
       " 'This is a effusion thing.',\n",
       " 'This is a effusive thing.',\n",
       " 'This is a effusively thing.',\n",
       " 'This is a effusiveness thing.',\n",
       " 'This is a elan thing.',\n",
       " 'This is a elate thing.',\n",
       " 'This is a elated thing.',\n",
       " 'This is a elatedly thing.',\n",
       " 'This is a elation thing.',\n",
       " 'This is a electrify thing.',\n",
       " 'This is a elegance thing.',\n",
       " 'This is a elegant thing.',\n",
       " 'This is a elegantly thing.',\n",
       " 'This is a elevate thing.',\n",
       " 'This is a elite thing.',\n",
       " 'This is a eloquence thing.',\n",
       " 'This is a eloquent thing.',\n",
       " 'This is a eloquently thing.',\n",
       " 'This is a embolden thing.',\n",
       " 'This is a eminence thing.',\n",
       " 'This is a eminent thing.',\n",
       " 'This is a empathize thing.',\n",
       " 'This is a empathy thing.',\n",
       " 'This is a empower thing.',\n",
       " 'This is a empowerment thing.',\n",
       " 'This is a enchant thing.',\n",
       " 'This is a enchanted thing.',\n",
       " 'This is a enchanting thing.',\n",
       " 'This is a enchantingly thing.',\n",
       " 'This is a encourage thing.',\n",
       " 'This is a encouragement thing.',\n",
       " 'This is a encouraging thing.',\n",
       " 'This is a encouragingly thing.',\n",
       " 'This is a endear thing.',\n",
       " 'This is a endearing thing.',\n",
       " 'This is a endorse thing.',\n",
       " 'This is a endorsed thing.',\n",
       " 'This is a endorsement thing.',\n",
       " 'This is a endorses thing.',\n",
       " 'This is a endorsing thing.',\n",
       " 'This is a energetic thing.',\n",
       " 'This is a energize thing.',\n",
       " 'This is a energy-efficient thing.',\n",
       " 'This is a energy-saving thing.',\n",
       " 'This is a engaging thing.',\n",
       " 'This is a engrossing thing.',\n",
       " 'This is a enhance thing.',\n",
       " 'This is a enhanced thing.',\n",
       " 'This is a enhancement thing.',\n",
       " 'This is a enhances thing.',\n",
       " 'This is a enjoy thing.',\n",
       " 'This is a enjoyable thing.',\n",
       " 'This is a enjoyably thing.',\n",
       " 'This is a enjoyed thing.',\n",
       " 'This is a enjoying thing.',\n",
       " 'This is a enjoyment thing.',\n",
       " 'This is a enjoys thing.',\n",
       " 'This is a enlighten thing.',\n",
       " 'This is a enlightenment thing.',\n",
       " 'This is a enliven thing.',\n",
       " 'This is a ennoble thing.',\n",
       " 'This is a enough thing.',\n",
       " 'This is a enrapt thing.',\n",
       " 'This is a enrapture thing.',\n",
       " 'This is a enraptured thing.',\n",
       " 'This is a enrich thing.',\n",
       " 'This is a enrichment thing.',\n",
       " 'This is a enterprising thing.',\n",
       " 'This is a entertain thing.',\n",
       " 'This is a entertaining thing.',\n",
       " 'This is a entertains thing.',\n",
       " 'This is a enthral thing.',\n",
       " 'This is a enthrall thing.',\n",
       " 'This is a enthralled thing.',\n",
       " 'This is a enthuse thing.',\n",
       " 'This is a enthusiasm thing.',\n",
       " 'This is a enthusiast thing.',\n",
       " 'This is a enthusiastic thing.',\n",
       " 'This is a enthusiastically thing.',\n",
       " 'This is a entice thing.',\n",
       " 'This is a enticed thing.',\n",
       " 'This is a enticing thing.',\n",
       " 'This is a enticingly thing.',\n",
       " 'This is a entranced thing.',\n",
       " 'This is a entrancing thing.',\n",
       " 'This is a entrust thing.',\n",
       " 'This is a enviable thing.',\n",
       " 'This is a enviably thing.',\n",
       " 'This is a envious thing.',\n",
       " 'This is a enviously thing.',\n",
       " 'This is a enviousness thing.',\n",
       " 'This is a envy thing.',\n",
       " 'This is a equitable thing.',\n",
       " 'This is a ergonomical thing.',\n",
       " 'This is a err-free thing.',\n",
       " 'This is a erudite thing.',\n",
       " 'This is a ethical thing.',\n",
       " 'This is a eulogize thing.',\n",
       " 'This is a euphoria thing.',\n",
       " 'This is a euphoric thing.',\n",
       " 'This is a euphorically thing.',\n",
       " 'This is a evaluative thing.',\n",
       " 'This is a evenly thing.',\n",
       " 'This is a eventful thing.',\n",
       " 'This is a everlasting thing.',\n",
       " 'This is a evocative thing.',\n",
       " 'This is a exalt thing.',\n",
       " 'This is a exaltation thing.',\n",
       " 'This is a exalted thing.',\n",
       " 'This is a exaltedly thing.',\n",
       " 'This is a exalting thing.',\n",
       " 'This is a exaltingly thing.',\n",
       " 'This is a examplar thing.',\n",
       " 'This is a examplary thing.',\n",
       " 'This is a excallent thing.',\n",
       " 'This is a exceed thing.',\n",
       " 'This is a exceeded thing.',\n",
       " 'This is a exceeding thing.',\n",
       " 'This is a exceedingly thing.',\n",
       " 'This is a exceeds thing.',\n",
       " 'This is a excel thing.',\n",
       " 'This is a exceled thing.',\n",
       " 'This is a excelent thing.',\n",
       " 'This is a excellant thing.',\n",
       " 'This is a excelled thing.',\n",
       " 'This is a excellence thing.',\n",
       " 'This is a excellency thing.',\n",
       " 'This is a excellent thing.',\n",
       " 'This is a excellently thing.',\n",
       " 'This is a excels thing.',\n",
       " 'This is a exceptional thing.',\n",
       " 'This is a exceptionally thing.',\n",
       " 'This is a excite thing.',\n",
       " 'This is a excited thing.',\n",
       " 'This is a excitedly thing.',\n",
       " 'This is a excitedness thing.',\n",
       " 'This is a excitement thing.',\n",
       " 'This is a excites thing.',\n",
       " 'This is a exciting thing.',\n",
       " 'This is a excitingly thing.',\n",
       " 'This is a exellent thing.',\n",
       " 'This is a exemplar thing.',\n",
       " 'This is a exemplary thing.',\n",
       " 'This is a exhilarate thing.',\n",
       " 'This is a exhilarating thing.',\n",
       " 'This is a exhilaratingly thing.',\n",
       " 'This is a exhilaration thing.',\n",
       " 'This is a exonerate thing.',\n",
       " 'This is a expansive thing.',\n",
       " 'This is a expeditiously thing.',\n",
       " 'This is a expertly thing.',\n",
       " 'This is a exquisite thing.',\n",
       " 'This is a exquisitely thing.',\n",
       " 'This is a extol thing.',\n",
       " 'This is a extoll thing.',\n",
       " 'This is a extraordinarily thing.',\n",
       " 'This is a extraordinary thing.',\n",
       " 'This is a exuberance thing.',\n",
       " 'This is a exuberant thing.',\n",
       " 'This is a exuberantly thing.',\n",
       " 'This is a exult thing.',\n",
       " 'This is a exultant thing.',\n",
       " 'This is a exultation thing.',\n",
       " 'This is a exultingly thing.',\n",
       " 'This is a eye-catch thing.',\n",
       " 'This is a eye-catching thing.',\n",
       " 'This is a eyecatch thing.',\n",
       " 'This is a eyecatching thing.',\n",
       " 'This is a fabulous thing.',\n",
       " 'This is a fabulously thing.',\n",
       " 'This is a facilitate thing.',\n",
       " 'This is a fair thing.',\n",
       " 'This is a fairly thing.',\n",
       " 'This is a fairness thing.',\n",
       " 'This is a faith thing.',\n",
       " 'This is a faithful thing.',\n",
       " 'This is a faithfully thing.',\n",
       " 'This is a faithfulness thing.',\n",
       " 'This is a fame thing.',\n",
       " 'This is a famed thing.',\n",
       " 'This is a famous thing.',\n",
       " 'This is a famously thing.',\n",
       " 'This is a fancier thing.',\n",
       " 'This is a fancinating thing.',\n",
       " 'This is a fancy thing.',\n",
       " 'This is a fanfare thing.',\n",
       " 'This is a fans thing.',\n",
       " 'This is a fantastic thing.',\n",
       " 'This is a fantastically thing.',\n",
       " 'This is a fascinate thing.',\n",
       " 'This is a fascinating thing.',\n",
       " 'This is a fascinatingly thing.',\n",
       " 'This is a fascination thing.',\n",
       " 'This is a fashionable thing.',\n",
       " 'This is a fashionably thing.',\n",
       " 'This is a fast thing.',\n",
       " 'This is a fast-growing thing.',\n",
       " 'This is a fast-paced thing.',\n",
       " 'This is a faster thing.',\n",
       " 'This is a fastest thing.',\n",
       " 'This is a fastest-growing thing.',\n",
       " 'This is a faultless thing.',\n",
       " 'This is a fav thing.',\n",
       " 'This is a fave thing.',\n",
       " 'This is a favor thing.',\n",
       " 'This is a favorable thing.',\n",
       " 'This is a favored thing.',\n",
       " 'This is a favorite thing.',\n",
       " 'This is a favorited thing.',\n",
       " 'This is a favour thing.',\n",
       " 'This is a fearless thing.',\n",
       " 'This is a fearlessly thing.',\n",
       " 'This is a feasible thing.',\n",
       " 'This is a feasibly thing.',\n",
       " 'This is a feat thing.',\n",
       " 'This is a feature-rich thing.',\n",
       " 'This is a fecilitous thing.',\n",
       " 'This is a feisty thing.',\n",
       " 'This is a felicitate thing.',\n",
       " 'This is a felicitous thing.',\n",
       " 'This is a felicity thing.',\n",
       " 'This is a fertile thing.',\n",
       " 'This is a fervent thing.',\n",
       " 'This is a fervently thing.',\n",
       " 'This is a fervid thing.',\n",
       " 'This is a fervidly thing.',\n",
       " 'This is a fervor thing.',\n",
       " 'This is a festive thing.',\n",
       " 'This is a fidelity thing.',\n",
       " 'This is a fiery thing.',\n",
       " 'This is a fine thing.',\n",
       " 'This is a fine-looking thing.',\n",
       " 'This is a finely thing.',\n",
       " 'This is a finer thing.',\n",
       " 'This is a finest thing.',\n",
       " 'This is a firmer thing.',\n",
       " 'This is a first-class thing.',\n",
       " 'This is a first-in-class thing.',\n",
       " 'This is a first-rate thing.',\n",
       " 'This is a flashy thing.',\n",
       " 'This is a flatter thing.',\n",
       " 'This is a flattering thing.',\n",
       " 'This is a flatteringly thing.',\n",
       " 'This is a flawless thing.',\n",
       " 'This is a flawlessly thing.',\n",
       " 'This is a flexibility thing.',\n",
       " 'This is a flexible thing.',\n",
       " 'This is a flourish thing.',\n",
       " 'This is a flourishing thing.',\n",
       " 'This is a fluent thing.',\n",
       " 'This is a flutter thing.',\n",
       " 'This is a fond thing.',\n",
       " 'This is a fondly thing.',\n",
       " 'This is a fondness thing.',\n",
       " 'This is a foolproof thing.',\n",
       " 'This is a foremost thing.',\n",
       " 'This is a foresight thing.',\n",
       " 'This is a formidable thing.',\n",
       " 'This is a fortitude thing.',\n",
       " 'This is a fortuitous thing.',\n",
       " 'This is a fortuitously thing.',\n",
       " 'This is a fortunate thing.',\n",
       " 'This is a fortunately thing.',\n",
       " 'This is a fortune thing.',\n",
       " 'This is a fragrant thing.',\n",
       " 'This is a free thing.',\n",
       " 'This is a freed thing.',\n",
       " 'This is a freedom thing.',\n",
       " 'This is a freedoms thing.',\n",
       " 'This is a fresh thing.',\n",
       " 'This is a fresher thing.',\n",
       " 'This is a freshest thing.',\n",
       " 'This is a friendliness thing.',\n",
       " 'This is a friendly thing.',\n",
       " 'This is a frolic thing.',\n",
       " 'This is a frugal thing.',\n",
       " 'This is a fruitful thing.',\n",
       " 'This is a ftw thing.',\n",
       " 'This is a fulfillment thing.',\n",
       " 'This is a fun thing.',\n",
       " 'This is a futurestic thing.',\n",
       " 'This is a futuristic thing.',\n",
       " 'This is a gaiety thing.',\n",
       " 'This is a gaily thing.',\n",
       " 'This is a gain thing.',\n",
       " 'This is a gained thing.',\n",
       " 'This is a gainful thing.',\n",
       " 'This is a gainfully thing.',\n",
       " 'This is a gaining thing.',\n",
       " 'This is a gains thing.',\n",
       " 'This is a gallant thing.',\n",
       " 'This is a gallantly thing.',\n",
       " 'This is a galore thing.',\n",
       " 'This is a geekier thing.',\n",
       " 'This is a geeky thing.',\n",
       " 'This is a gem thing.',\n",
       " 'This is a gems thing.',\n",
       " 'This is a generosity thing.',\n",
       " 'This is a generous thing.',\n",
       " 'This is a generously thing.',\n",
       " 'This is a genial thing.',\n",
       " 'This is a genius thing.',\n",
       " 'This is a gentle thing.',\n",
       " 'This is a gentlest thing.',\n",
       " 'This is a genuine thing.',\n",
       " 'This is a gifted thing.',\n",
       " 'This is a glad thing.',\n",
       " 'This is a gladden thing.',\n",
       " 'This is a gladly thing.',\n",
       " 'This is a gladness thing.',\n",
       " 'This is a glamorous thing.',\n",
       " 'This is a glee thing.',\n",
       " 'This is a gleeful thing.',\n",
       " 'This is a gleefully thing.',\n",
       " 'This is a glimmer thing.',\n",
       " 'This is a glimmering thing.',\n",
       " 'This is a glisten thing.',\n",
       " 'This is a glistening thing.',\n",
       " 'This is a glitter thing.',\n",
       " 'This is a glitz thing.',\n",
       " 'This is a glorify thing.',\n",
       " 'This is a glorious thing.',\n",
       " 'This is a gloriously thing.',\n",
       " 'This is a glory thing.',\n",
       " 'This is a glow thing.',\n",
       " 'This is a glowing thing.',\n",
       " 'This is a glowingly thing.',\n",
       " 'This is a god-given thing.',\n",
       " 'This is a god-send thing.',\n",
       " 'This is a godlike thing.',\n",
       " 'This is a godsend thing.',\n",
       " 'This is a gold thing.',\n",
       " 'This is a golden thing.',\n",
       " 'This is a good thing.',\n",
       " 'This is a goodly thing.',\n",
       " 'This is a goodness thing.',\n",
       " 'This is a goodwill thing.',\n",
       " 'This is a goood thing.',\n",
       " 'This is a gooood thing.',\n",
       " 'This is a gorgeous thing.',\n",
       " 'This is a gorgeously thing.',\n",
       " 'This is a grace thing.',\n",
       " 'This is a graceful thing.',\n",
       " 'This is a gracefully thing.',\n",
       " 'This is a gracious thing.',\n",
       " 'This is a graciously thing.',\n",
       " 'This is a graciousness thing.',\n",
       " 'This is a grand thing.',\n",
       " 'This is a grandeur thing.',\n",
       " 'This is a grateful thing.',\n",
       " 'This is a gratefully thing.',\n",
       " 'This is a gratification thing.',\n",
       " 'This is a gratified thing.',\n",
       " 'This is a gratifies thing.',\n",
       " 'This is a gratify thing.',\n",
       " 'This is a gratifying thing.',\n",
       " 'This is a gratifyingly thing.',\n",
       " 'This is a gratitude thing.',\n",
       " 'This is a great thing.',\n",
       " 'This is a greatest thing.',\n",
       " 'This is a greatness thing.',\n",
       " 'This is a grin thing.',\n",
       " 'This is a groundbreaking thing.',\n",
       " 'This is a guarantee thing.',\n",
       " 'This is a guidance thing.',\n",
       " 'This is a guiltless thing.',\n",
       " 'This is a gumption thing.',\n",
       " 'This is a gush thing.',\n",
       " 'This is a gusto thing.',\n",
       " 'This is a gutsy thing.',\n",
       " 'This is a hail thing.',\n",
       " 'This is a halcyon thing.',\n",
       " 'This is a hale thing.',\n",
       " 'This is a hallmark thing.',\n",
       " 'This is a hallmarks thing.',\n",
       " 'This is a hallowed thing.',\n",
       " 'This is a handier thing.',\n",
       " 'This is a handily thing.',\n",
       " 'This is a hands-down thing.',\n",
       " 'This is a handsome thing.',\n",
       " 'This is a handsomely thing.',\n",
       " 'This is a handy thing.',\n",
       " 'This is a happier thing.',\n",
       " 'This is a happily thing.',\n",
       " 'This is a happiness thing.',\n",
       " 'This is a happy thing.',\n",
       " 'This is a hard-working thing.',\n",
       " 'This is a hardier thing.',\n",
       " 'This is a hardy thing.',\n",
       " 'This is a harmless thing.',\n",
       " 'This is a harmonious thing.',\n",
       " 'This is a harmoniously thing.',\n",
       " 'This is a harmonize thing.',\n",
       " 'This is a harmony thing.',\n",
       " 'This is a headway thing.',\n",
       " 'This is a heal thing.',\n",
       " 'This is a healthful thing.',\n",
       " 'This is a healthy thing.',\n",
       " 'This is a hearten thing.',\n",
       " 'This is a heartening thing.',\n",
       " 'This is a heartfelt thing.',\n",
       " 'This is a heartily thing.',\n",
       " 'This is a heartwarming thing.',\n",
       " 'This is a heaven thing.',\n",
       " 'This is a heavenly thing.',\n",
       " 'This is a helped thing.',\n",
       " 'This is a helpful thing.',\n",
       " 'This is a helping thing.',\n",
       " 'This is a hero thing.',\n",
       " 'This is a heroic thing.',\n",
       " 'This is a heroically thing.',\n",
       " 'This is a heroine thing.',\n",
       " 'This is a heroize thing.',\n",
       " 'This is a heros thing.',\n",
       " 'This is a high-quality thing.',\n",
       " 'This is a high-spirited thing.',\n",
       " 'This is a hilarious thing.',\n",
       " 'This is a holy thing.',\n",
       " 'This is a homage thing.',\n",
       " 'This is a honest thing.',\n",
       " 'This is a honesty thing.',\n",
       " 'This is a honor thing.',\n",
       " 'This is a honorable thing.',\n",
       " 'This is a honored thing.',\n",
       " 'This is a honoring thing.',\n",
       " 'This is a hooray thing.',\n",
       " 'This is a hopeful thing.',\n",
       " 'This is a hospitable thing.',\n",
       " 'This is a hot thing.',\n",
       " 'This is a hotcake thing.',\n",
       " 'This is a hotcakes thing.',\n",
       " 'This is a hottest thing.',\n",
       " 'This is a hug thing.',\n",
       " 'This is a humane thing.',\n",
       " 'This is a humble thing.',\n",
       " 'This is a humility thing.',\n",
       " 'This is a humor thing.',\n",
       " 'This is a humorous thing.',\n",
       " 'This is a humorously thing.',\n",
       " 'This is a humour thing.',\n",
       " 'This is a humourous thing.',\n",
       " 'This is a ideal thing.',\n",
       " 'This is a idealize thing.',\n",
       " 'This is a ideally thing.',\n",
       " 'This is a idol thing.',\n",
       " 'This is a idolize thing.',\n",
       " 'This is a idolized thing.',\n",
       " 'This is a idyllic thing.',\n",
       " 'This is a illuminate thing.',\n",
       " 'This is a illuminati thing.',\n",
       " 'This is a illuminating thing.',\n",
       " 'This is a illumine thing.',\n",
       " 'This is a illustrious thing.',\n",
       " 'This is a ilu thing.',\n",
       " 'This is a imaculate thing.',\n",
       " 'This is a imaginative thing.',\n",
       " 'This is a immaculate thing.',\n",
       " 'This is a immaculately thing.',\n",
       " 'This is a immense thing.',\n",
       " 'This is a impartial thing.',\n",
       " 'This is a impartiality thing.',\n",
       " 'This is a impartially thing.',\n",
       " 'This is a impassioned thing.',\n",
       " 'This is a impeccable thing.',\n",
       " 'This is a impeccably thing.',\n",
       " 'This is a important thing.',\n",
       " 'This is a impress thing.',\n",
       " 'This is a impressed thing.',\n",
       " 'This is a impresses thing.',\n",
       " 'This is a impressive thing.',\n",
       " 'This is a impressively thing.',\n",
       " 'This is a impressiveness thing.',\n",
       " 'This is a improve thing.',\n",
       " 'This is a improved thing.',\n",
       " 'This is a improvement thing.',\n",
       " 'This is a improvements thing.',\n",
       " 'This is a improves thing.',\n",
       " 'This is a improving thing.',\n",
       " 'This is a incredible thing.',\n",
       " 'This is a incredibly thing.',\n",
       " 'This is a indebted thing.',\n",
       " 'This is a individualized thing.',\n",
       " 'This is a indulgence thing.',\n",
       " 'This is a indulgent thing.',\n",
       " 'This is a industrious thing.',\n",
       " 'This is a inestimable thing.',\n",
       " 'This is a inestimably thing.',\n",
       " 'This is a inexpensive thing.',\n",
       " 'This is a infallibility thing.',\n",
       " 'This is a infallible thing.',\n",
       " 'This is a infallibly thing.',\n",
       " 'This is a influential thing.',\n",
       " 'This is a ingenious thing.',\n",
       " 'This is a ingeniously thing.',\n",
       " 'This is a ingenuity thing.',\n",
       " 'This is a ingenuous thing.',\n",
       " 'This is a ingenuously thing.',\n",
       " 'This is a innocuous thing.',\n",
       " 'This is a innovation thing.',\n",
       " 'This is a innovative thing.',\n",
       " 'This is a inpressed thing.',\n",
       " 'This is a insightful thing.',\n",
       " ...]"
      ]
     },
     "execution_count": 608,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is a {pos} thing.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = [x.strip() for x in open('/tmp/positive-words.txt').readlines()]\n",
    "neg = [x.strip() for x in open('/tmp/negative-words.txt').readlines()]\n",
    "editor.tg.unmask('These seats are <mask>.', candidates=['Ġ' + x for x in neg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mltests import common_things\n",
    "common = common_things.CommonThings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 529,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "first = [x for y in zip(men[:100], women) for x in y][:100]\n",
    "first_and_pronoun= [{'name': x[0], 'pronoun': x[1]} for y in zip([(a, 'he') for a in men], [(a, 'she') for a in women]) for x in y][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first_and_pronoun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [],
   "source": [
    "men = [x for x in common.names['first_sorted'] if common.names['first'][x]['sex'] == 'M' and common.names['first'][x]['count'] > 500]\n",
    "women = [x for x in common.names['first_sorted'] if common.names['first'][x]['sex'] == 'F' and common.names['first'][x]['count'] > 500]\n",
    "last = [x for x in common.names['last_sorted'] if common.names['last'][x]['count'] > 3000]\n",
    "first = [x for y in zip(men[:100], women) for x in y][:100]\n",
    "first_and_pronoun= [{'name': x[0], 'pronoun': x[1]} for y in zip([(a, 'he') for a in men], [(a, 'she') for a in women]) for x in y][:100]\n",
    "\n",
    "countries = common.get_countries(n=1000000)\n",
    "nationalities = common.get_nationalities(n=1000000)\n",
    "cities = common.cities\n",
    "religions = common.get_religions()\n",
    "religion_adj = common.get_religions(adj=True)\n",
    "sexual_adj = common.get_sexual_adjs()\n",
    "json.dump( {\n",
    "    'male': men[:100],\n",
    "    'female': women[:100],\n",
    "    'first_name': first,\n",
    "    'first_pronoun': first_and_pronoun,\n",
    "    'last_name': last[:100],\n",
    "    'country': countries,\n",
    "    'nationality': nationalities,\n",
    "    'city': cities,\n",
    "    'religion': religions,\n",
    "    'religion_adj': religion_adj,\n",
    "    'sexual_adj': sexual_adj,\n",
    "}, open('/home/marcotcr/work/checklist/data/lexicons/basic.json', 'w'))\n",
    "json.dump({\n",
    "    'men': men,\n",
    "    'women': women,\n",
    "    'last': last,\n",
    "},open('/home/marcotcr/work/checklist/data/names.json', 'w'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Chinese',\n",
       " 'Indian',\n",
       " 'American',\n",
       " 'Indonesian',\n",
       " 'Pakistani',\n",
       " 'Brazilian',\n",
       " 'Nigerian',\n",
       " 'Bangladeshi',\n",
       " 'Russian',\n",
       " 'Japanese',\n",
       " 'Mexican',\n",
       " 'Ethiopian',\n",
       " 'Philippine',\n",
       " 'Egyptian',\n",
       " 'Vietnamese',\n",
       " 'German',\n",
       " 'Turkish',\n",
       " 'Iranian',\n",
       " 'Thai',\n",
       " 'French',\n",
       " 'British',\n",
       " 'Italian',\n",
       " 'South African',\n",
       " 'Tanzanian',\n",
       " 'Burmese',\n",
       " 'Kenyan',\n",
       " 'Colombian',\n",
       " 'Spanish',\n",
       " 'Ukrainian',\n",
       " 'Argentine',\n",
       " 'Ugandan',\n",
       " 'Algerian',\n",
       " 'Sudanese',\n",
       " 'Iraqi',\n",
       " 'Polish',\n",
       " 'Afghan',\n",
       " 'Canadian',\n",
       " 'Moroccan',\n",
       " 'Saudi',\n",
       " 'Uzbekistani',\n",
       " 'Peruvian',\n",
       " 'Malaysian',\n",
       " 'Angolan',\n",
       " 'Ghanaian',\n",
       " 'Mozambican',\n",
       " 'Venezuelan',\n",
       " 'Yemeni',\n",
       " 'Nepali',\n",
       " 'Malagasy',\n",
       " 'South Korean',\n",
       " 'Cameroonian',\n",
       " 'Australian',\n",
       " 'Nigerien',\n",
       " 'Sri Lankan',\n",
       " 'Burkinabé',\n",
       " 'Romanian',\n",
       " 'Malian',\n",
       " 'Chilean',\n",
       " 'Kazakhstani',\n",
       " 'Malawian',\n",
       " 'Zambian',\n",
       " 'Guatemalan',\n",
       " 'Dutch',\n",
       " 'Ecuadorian',\n",
       " 'Syrian',\n",
       " 'Cambodian',\n",
       " 'Senegalese',\n",
       " 'Chadian',\n",
       " 'Somali',\n",
       " 'Zimbabwean',\n",
       " 'Guinean',\n",
       " 'Rwandan',\n",
       " 'Tunisian',\n",
       " 'Beninese',\n",
       " 'Belgian',\n",
       " 'Bolivian',\n",
       " 'Cuban',\n",
       " 'Burundian',\n",
       " 'Haitian',\n",
       " 'South Sudanese',\n",
       " 'Greek',\n",
       " 'Dominican',\n",
       " 'Czech',\n",
       " 'Portuguese',\n",
       " 'Swedish',\n",
       " 'Jordanian',\n",
       " 'Azerbaijani',\n",
       " 'Hungarian',\n",
       " 'Emirati',\n",
       " 'Honduran',\n",
       " 'Belarusian',\n",
       " 'Tajikistani',\n",
       " 'Israeli',\n",
       " 'Austrian',\n",
       " 'Papua New Guinean',\n",
       " 'Swiss',\n",
       " 'Togolese',\n",
       " 'Sierra Leonean',\n",
       " 'Hong Kong',\n",
       " 'Lao',\n",
       " 'Bulgarian',\n",
       " 'Serbian',\n",
       " 'Paraguayan',\n",
       " 'Lebanese',\n",
       " 'Libyan',\n",
       " 'Nicaraguan',\n",
       " 'Salvadoran',\n",
       " 'Kyrgyzstani',\n",
       " 'Turkmen',\n",
       " 'Danish',\n",
       " 'Singaporean',\n",
       " 'Finnish',\n",
       " 'Slovak',\n",
       " 'Norwegian',\n",
       " 'Congolese',\n",
       " 'Costa Rican',\n",
       " 'New Zealand',\n",
       " 'Irish',\n",
       " 'Omani',\n",
       " 'Liberian',\n",
       " 'Central African',\n",
       " 'Palestinian',\n",
       " 'Mauritanian',\n",
       " 'Panamanian',\n",
       " 'Kuwaiti',\n",
       " 'Croatian',\n",
       " 'Georgian',\n",
       " 'Moldovan',\n",
       " 'Uruguayan',\n",
       " 'Bosnian or Herzegovinian',\n",
       " 'Eritrean',\n",
       " 'Puerto Rican',\n",
       " 'Mongolian',\n",
       " 'Armenian',\n",
       " 'Jamaican',\n",
       " 'Albanian',\n",
       " 'Lithuanian',\n",
       " 'Qatari',\n",
       " 'Namibian',\n",
       " 'Gambian',\n",
       " 'Motswana',\n",
       " 'Gabonese',\n",
       " 'Basotho',\n",
       " 'Macedonian',\n",
       " 'Slovenian',\n",
       " 'Latvian',\n",
       " 'Bissau-Guinean',\n",
       " 'from Kosovo',\n",
       " 'Bahraini',\n",
       " 'Trinidadian or Tobagonian',\n",
       " 'Estonian',\n",
       " 'Equatorial Guinean',\n",
       " 'Timorese',\n",
       " 'Mauritian',\n",
       " 'Cypriot',\n",
       " 'Swazi',\n",
       " 'Djiboutian',\n",
       " 'Fijian',\n",
       " 'Comoran',\n",
       " 'Guyanese',\n",
       " 'Bhutanese',\n",
       " 'Solomon Island',\n",
       " 'Macanese',\n",
       " 'Montenegrin',\n",
       " 'Luxembourg',\n",
       " 'Surinamese',\n",
       " 'Cabo Verdean',\n",
       " 'Maldivian',\n",
       " 'Maltese',\n",
       " 'Bruneian',\n",
       " 'Bahamian',\n",
       " 'Belizean',\n",
       " 'Icelandic',\n",
       " 'Ni-Vanuatu',\n",
       " 'Barbadian',\n",
       " 'New Caledonian',\n",
       " 'French Polynesian',\n",
       " 'Samoan',\n",
       " 'Saint Lucian',\n",
       " 'from Channel Islands',\n",
       " 'Guamanian',\n",
       " 'I-Kiribati',\n",
       " 'Micronesian',\n",
       " 'Grenadian',\n",
       " 'Saint Vincentian',\n",
       " 'U.S. Virgin Island',\n",
       " 'Aruban',\n",
       " 'Tongan',\n",
       " 'Seychellois',\n",
       " 'Antiguan or Barbudan',\n",
       " 'Manx',\n",
       " 'Andorran',\n",
       " 'Dominican',\n",
       " 'Caymanian',\n",
       " 'Bermudian',\n",
       " 'Marshallese',\n",
       " 'Northern Marianan',\n",
       " 'Greenlandic',\n",
       " 'American Samoan',\n",
       " 'Kittitian or Nevisian',\n",
       " 'Faroese',\n",
       " 'Sint Maarten',\n",
       " 'Monégasque',\n",
       " 'Liechtenstein',\n",
       " 'Turks and Caicos Island',\n",
       " 'Saint-Martinoise',\n",
       " 'Sammarinese',\n",
       " 'Gibraltar',\n",
       " 'British Virgin Island',\n",
       " 'Palauan',\n",
       " 'Nauruan',\n",
       " 'Tuvaluan',\n",
       " 'Ivorian',\n",
       " 'Curaçaoan',\n",
       " 'São Toméan']"
      ]
     },
     "execution_count": 485,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 482,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(common.cities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['female_names', 'male_names', 'names', 'last_names', 'countries', 'nationalities', 'cities', 'sexuality_adjs', 'religions', 'religion_adjs', 'verbs_3ps', 'verbs_3pp', 'nouns', 'adjs'])"
      ]
     },
     "execution_count": 424,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "editor = checklist.editor.Editor()\n",
    "c = pickle.load(open('/home/marcotcr/work/ml-tests/data/common.pkl', 'rb'))\n",
    "editor.lexicons['male'] = c['male_names']\n",
    "editor.lexicons['female'] = c['female_names']\n",
    "c.keys()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['new',\n",
       " 'old',\n",
       " 'good',\n",
       " 'bad',\n",
       " 'great',\n",
       " 'big',\n",
       " 'small',\n",
       " 'white',\n",
       " 'black',\n",
       " 'red',\n",
       " 'blue',\n",
       " 'yellow']"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c['adjs']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Editing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'tuple' object has no attribute 'replace'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-81-b2b713fc6478>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0meditor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'This is not {bert}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'This is not {bert} bad'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/work/checklist/checklist/editor.py\u001b[0m in \u001b[0;36mtemplate\u001b[0;34m(self, templates, return_meta, nsamples, product, remove_duplicates, **kwargs)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0;31m# print(ts)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsamples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremove_duplicates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mremove_duplicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtok\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmask_token\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msamp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             \u001b[0;31m# print(samp)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munmask_multiple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeam_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/work/checklist/checklist/editor.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0;31m# print(ts)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsamples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremove_duplicates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mremove_duplicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtok\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmask_token\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msamp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             \u001b[0;31m# print(samp)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munmask_multiple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeam_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'replace'"
     ]
    }
   ],
   "source": [
    "editor.template(('This is not {bert}', 'This is not {bert} bad'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('This is not VERYLONGTOKENTHATWILLNOTEXISTEVER VERYLONGTOKENTHATWILLNOTEXISTEVER', 'This is not VERYLONGTOKENTHATWILLNOTEXISTEVER bad')\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'x' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-61-3b009b295e57>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0meditor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'This is not {bert} {bert}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'This is not {bert} bad'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/work/checklist/checklist/editor.py\u001b[0m in \u001b[0;36mtemplate\u001b[0;34m(self, templates, return_meta, nsamples, product, remove_duplicates, **kwargs)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsamples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremove_duplicates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mremove_duplicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m             \u001b[0msamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtok\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbert_tokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmask_token\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m             \u001b[0;31m# self.tg.unmask_multiple(samp,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'x' is not defined"
     ]
    }
   ],
   "source": [
    "editor.template(('This is not {bert} {bert}', 'This is not {bert} bad'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cat dog dog'"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub('dog', 'cat', 'dog dog dog', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is not {bert[0]} {bert[1]} {bert1[2]} {bert2[3]}'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist.editor.replace_bert('This is not {bert} {bert} {bert1} {bert2}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = re.compile(r'\\{bert\\d*\\}')\n",
    "a.findall('This is not {bert} {bert1}, {bert342}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Can only have one bert index per template string",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-116-ba59f54f1490>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mchecklist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meditor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_bert_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'This is not {besrt} {abert} {bert} {bert2}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/work/checklist/checklist/editor.py\u001b[0m in \u001b[0;36mget_bert_index\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mberts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbert_finder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfindall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mberts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Can only have one bert index per template string'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mberts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0mret\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbert_rep\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mberts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mException\u001b[0m: Can only have one bert index per template string"
     ]
    }
   ],
   "source": [
    "checklist.editor.get_bert_index(('This is not {besrt} {abert} {bert} {bert2}'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function checklist.editor.bert_index.<locals>.<lambda>()>,\n",
       "            {'bert': ['This is not {bert} {bert}', 'This is not {bert} bad'],\n",
       "             'bert1': ['This is not {bert1} s']})"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checklist.editor.bert_index(('This is not {bert} {bert}', 'This is not {bert} bad', 'This is not {bert1} s'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is not a1 a2.', 'This is not b1 b2.']"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is not {b[0]} {b[1]}.', b=[('a1', 'a2'), ('b1', 'b2')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = editor.tg.unmask_multiple(['This is a <mask>.', 'This is a <mask> <mask>.'], beam_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['short', 'version'], ' This is a short version.', 13.087730884552002)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.tg.unmask_multiple(['This is <mask>.', 'This is an <mask> <mask>.'], beam_size=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'str1': 'This is not the worst thing in the campus',\n",
       " 'str2': 'I think this is the worst place in the campus',\n",
       " 'str3': 'He is a good movie.'}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(['good'], ' This is not a very good movie.', 18.80685806274414),\n",
       " (['nice'], ' This is not a very nice movie.', 15.89453125),\n",
       " (['funny'], ' This is not a very funny movie.', 15.101058959960938),\n",
       " (['bad'], ' This is not a very bad movie.', 14.818763732910156),\n",
       " (['great'], ' This is not a very great movie.', 14.58511734008789),\n",
       " (['interesting'],\n",
       "  ' This is not a very interesting movie.',\n",
       "  14.357146263122559),\n",
       " (['fun'], ' This is not a very fun movie.', 14.004464149475098),\n",
       " (['exciting'], ' This is not a very exciting movie.', 13.874397277832031),\n",
       " (['pleasant'], ' This is not a very pleasant movie.', 13.860466003417969),\n",
       " (['long'], ' This is not a very long movie.', 13.80008316040039),\n",
       " (['violent'], ' This is not a very violent movie.', 13.722796440124512),\n",
       " (['entertaining'],\n",
       "  ' This is not a very entertaining movie.',\n",
       "  13.66596794128418),\n",
       " (['cool'], ' This is not a very cool movie.', 13.545897483825684),\n",
       " (['smart'], ' This is not a very smart movie.', 13.475765228271484),\n",
       " (['happy'], ' This is not a very happy movie.', 13.210590362548828),\n",
       " (['successful'], ' This is not a very successful movie.', 13.141807556152344),\n",
       " (['enjoyable'], ' This is not a very enjoyable movie.', 13.135686874389648),\n",
       " (['popular'], ' This is not a very popular movie.', 13.096426010131836),\n",
       " (['pretty'], ' This is not a very pretty movie.', 13.07435131072998),\n",
       " (['inspiring'], ' This is not a very inspiring movie.', 12.896029472351074)]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.tg.unmask('This is not a very <mask> movie.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(['player'], ' John is a very good player', 12.612285614013672),\n",
       " (['game'], ' John is a very good game', 12.206633567810059),\n",
       " (['book'], ' John is a very good book', 11.615123748779297),\n",
       " (['writer'], ' John is a very good writer', 11.266648292541504),\n",
       " (['example'], ' John is a very good example', 11.174124717712402),\n",
       " (['guy'], ' John is a very good guy', 11.041021347045898),\n",
       " (['read'], ' John is a very good read', 10.992713928222656),\n",
       " (['question'], ' John is a very good question', 10.974016189575195),\n",
       " (['pick'], ' John is a very good pick', 10.947866439819336),\n",
       " (['shooter'], ' John is a very good shooter', 10.893823623657227),\n",
       " (['story'], ' John is a very good story', 10.848344802856445),\n",
       " (['article'], ' John is a very good article', 10.771170616149902),\n",
       " (['candidate'], ' John is a very good candidate', 10.673225402832031),\n",
       " (['play'], ' John is a very good play', 10.646265029907227),\n",
       " (['idea'], ' John is a very good idea', 10.545669555664062),\n",
       " (['film'], ' John is a very good film', 10.3023681640625),\n",
       " (['coach'], ' John is a very good coach', 10.297456741333008),\n",
       " (['student'], ' John is a very good student', 10.237459182739258),\n",
       " (['list'], ' John is a very good list', 10.21300220489502),\n",
       " (['job'], ' John is a very good job', 10.210472106933594)]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.tg.unmask('John is a very good <mask>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is {a:thing} of {a:god}'"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "f = string.Formatter()\n",
    "s = 'This is {a:thing} of {a:god}'\n",
    "list(f.parse(s))\n",
    "checklist.editor.remove_options(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is {thing} of {god}'"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.sub(r'{\\1}', s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['thing', 'god']"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "m = re.compile(r'\\{[^\\}]+:([^\\}]*)\\}')\n",
    "m.findall(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this is a mna'"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_option = re.compile(r'.*:')\n",
    "remove_option.sub('', 'afsdf:this is a mna')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<function checklist.editor.get_bert_index.<locals>.<lambda>()>,\n",
       "            {'bert': ['This is {a:bert} {a:bert}.']})"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_finder = re.compile(r'\\{(?:[^\\}]*:)?bert\\d*\\}')\n",
    "x = 'This is {a:bert} {a:bert}.'\n",
    "checklist.editor.get_bert_index(x)\n",
    "# bert_finder.findall(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is {fsdfasfasd} {bert} {fsdfasfasd} {bert}.'"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'{a:([^}]*)}', r'{fsdfasfasd} {\\1}', x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is a bad man.',\n",
       " 'This is an incredible show.',\n",
       " 'This is a great person.',\n",
       " 'This is an important person.',\n",
       " 'This is an interesting person.']"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is {a:bert} {thing}.', thing=['movie', 'show', 'person', 'man'], nsamples=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'context': 'Samuel is not a feminist, Christine is.',\n",
       "  'question': 'Who is a feminist?',\n",
       "  'answer': 'Christine'},\n",
       " {'context': 'Stephen is not an adult, Ava is.',\n",
       "  'question': 'Who is an adult?',\n",
       "  'answer': 'Ava'},\n",
       " {'context': 'John is not an atheist, Angela is.',\n",
       "  'question': 'Who is an atheist?',\n",
       "  'answer': 'Angela'},\n",
       " {'context': 'Ethan is not a terrorist, Brooke is.',\n",
       "  'question': 'Who is a terrorist?',\n",
       "  'answer': 'Brooke'},\n",
       " {'context': 'Robert is not an adult, Sarah is.',\n",
       "  'question': 'Who is an adult?',\n",
       "  'answer': 'Sarah'}]"
      ]
     },
     "execution_count": 326,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template({\n",
    "   'context': '{male} is not {a:bert}, {female} is.' ,\n",
    "   'question': 'Who is {a:bert}?',\n",
    "   'answer': '{female}'\n",
    "    }, nsamples=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['John is a <mask> <mask>.', 'This is a <mask> <mask> man.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> movie.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> thing.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> person.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> thing.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> person.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> movie.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> person.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> thing.', 'John is a <mask> <mask>.', 'This is a <mask> <mask> thing.']\n",
      " 11.32  This is a real life thing.\n",
      " 10.31  This is a real world thing.\n",
      "  7.79  This is a good thing thing.\n",
      "  7.52  This is a very good thing.\n",
      "  7.52  This is a really good thing.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('real', 'life'),\n",
       " ('real', 'world'),\n",
       " ('good', 'thing'),\n",
       " ('very', 'good'),\n",
       " ('really', 'good'),\n",
       " ('really', 'cool'),\n",
       " ('very', 'interesting'),\n",
       " ('really', 'nice'),\n",
       " ('very', 'powerful'),\n",
       " ('very', 'important'),\n",
       " ('very', 'nice'),\n",
       " ('very', 'strange'),\n",
       " ('really', 'great'),\n",
       " ('really', 'bad'),\n",
       " ('very', 'sad'),\n",
       " ('pretty', 'cool'),\n",
       " ('really', 'interesting'),\n",
       " ('very', 'beautiful'),\n",
       " ('very', 'special'),\n",
       " ('really', 'funny'),\n",
       " ('very', 'serious'),\n",
       " ('pretty', 'good'),\n",
       " ('really', 'amazing'),\n",
       " ('really', 'beautiful'),\n",
       " ('truly', 'amazing'),\n",
       " ('really', 'scary'),\n",
       " ('really', 'strange'),\n",
       " ('truly', 'remarkable'),\n",
       " ('truly', 'wonderful'),\n",
       " ('pretty', 'scary'),\n",
       " ('pretty', 'amazing'),\n",
       " ('truly', 'great'),\n",
       " ('pretty', 'bad'),\n",
       " ('pretty', 'awesome'),\n",
       " ('pretty', 'interesting'),\n",
       " ('truly', 'beautiful'),\n",
       " ('free', 'trial'),\n",
       " ('pretty', 'funny'),\n",
       " ('truly', 'incredible'),\n",
       " ('pretty', 'nice'),\n",
       " ('free', 'transcript'),\n",
       " ('very', 'bad'),\n",
       " ('truly', 'extraordinary'),\n",
       " ('really', 'weird'),\n",
       " ('free', 'download'),\n",
       " ('science', 'writer'),\n",
       " ('truly', 'good'),\n",
       " ('pretty', 'strange'),\n",
       " ('very', 'scary'),\n",
       " ('dead', 'end'),\n",
       " ('very', 'complicated'),\n",
       " ('very', 'funny'),\n",
       " ('free', 'sample'),\n",
       " ('short', 'video'),\n",
       " ('rather', 'strange'),\n",
       " ('truly', 'special'),\n",
       " ('dead', 'man'),\n",
       " ('really', 'awesome'),\n",
       " ('personal', 'opinion'),\n",
       " ('short', 'version'),\n",
       " ('short', 'story'),\n",
       " ('free', 'bie'),\n",
       " ('truly', 'fantastic'),\n",
       " ('rather', 'interesting'),\n",
       " ('long', 'story'),\n",
       " ('political', 'scientist'),\n",
       " ('super', 'cool'),\n",
       " ('complete', 'transcript'),\n",
       " ('truly', 'powerful'),\n",
       " ('personal', 'favorite'),\n",
       " ('truly', 'phenomenal'),\n",
       " ('free', 'service'),\n",
       " ('free', 'event'),\n",
       " ('personal', 'essay'),\n",
       " ('really', 'sad'),\n",
       " ('really', 'wonderful'),\n",
       " ('good', 'idea'),\n",
       " ('free', 'demo'),\n",
       " ('free', 'product'),\n",
       " ('truly', 'magnificent'),\n",
       " ('science', 'journalist'),\n",
       " ('free', 'game'),\n",
       " ('personal', 'story'),\n",
       " ('really', 'terrible'),\n",
       " ('short', 'excerpt'),\n",
       " ('dead', 'horse'),\n",
       " ('free', 'article'),\n",
       " ('rather', 'odd'),\n",
       " ('pretty', 'weird'),\n",
       " ('dead', 'cat'),\n",
       " ('real', 'bad'),\n",
       " ('personal', 'experience'),\n",
       " ('free', 'speech'),\n",
       " ('real', 'good'),\n",
       " ('political', 'prisoner'),\n",
       " ('free', 'version'),\n",
       " ('rather', 'nice'),\n",
       " ('free', 'interview'),\n",
       " ('real', 'crime'),\n",
       " ('free', 'read'),\n",
       " ('rather', 'funny'),\n",
       " ('long', 'time'),\n",
       " ('short', 'summary'),\n",
       " ('personal', 'reflection'),\n",
       " ('science', 'fiction'),\n",
       " ('personal', 'note'),\n",
       " ('free', 'story'),\n",
       " ('science', 'nerd'),\n",
       " ('free', 'offering'),\n",
       " ('free', 'poll'),\n",
       " ('real', 'scary'),\n",
       " ('rather', 'unusual'),\n",
       " ('political', 'statement'),\n",
       " ('political', 'cartoon'),\n",
       " ('personal', 'review'),\n",
       " ('complete', 'list'),\n",
       " ('complete', 'summary'),\n",
       " ('complete', 'article'),\n",
       " ('short', 'clip'),\n",
       " ('free', 'commercial'),\n",
       " ('very', 'dangerous'),\n",
       " ('free', 'post'),\n",
       " ('free', 'resource'),\n",
       " ('personal', 'blog'),\n",
       " ('short', 'overview'),\n",
       " ('long', 'shot'),\n",
       " ('new', 'post'),\n",
       " ('pretty', 'stupid'),\n",
       " ('good', 'question'),\n",
       " ('science', 'reporter'),\n",
       " ('short', 'list'),\n",
       " ('short', 'post'),\n",
       " ('free', 'experiment'),\n",
       " ('good', 'day'),\n",
       " ('short', 'essay'),\n",
       " ('really', 'important'),\n",
       " ('short', 'film'),\n",
       " ('complete', 'interview'),\n",
       " ('dead', 'giveaway'),\n",
       " ('complete', 'post'),\n",
       " ('real', 'American'),\n",
       " ('free', 'broadcast'),\n",
       " ('free', 'demonstration'),\n",
       " ('dead', 'body'),\n",
       " ('complete', 'story'),\n",
       " ('free', 'video'),\n",
       " ('bad', 'idea'),\n",
       " ('short', 'article'),\n",
       " ('complete', 'breakdown'),\n",
       " ('personal', 'post'),\n",
       " ('good', 'start'),\n",
       " ('free', 'text'),\n",
       " ('recent', 'example'),\n",
       " ('different', 'story'),\n",
       " ('real', 'monster'),\n",
       " ('dead', 'bird'),\n",
       " ('short', 'primer'),\n",
       " ('free', 'encyclopedia'),\n",
       " ('short', 'explanation'),\n",
       " ('short', 'transcript'),\n",
       " ('dead', 'link'),\n",
       " ('conservative', 'Republican'),\n",
       " ('political', 'journalist'),\n",
       " ('long', 'list'),\n",
       " ('short', 'sample'),\n",
       " ('personal', 'view'),\n",
       " ('short', 'history'),\n",
       " ('good', 'example'),\n",
       " ('truly', 'awesome'),\n",
       " ('political', 'reporter'),\n",
       " ('dead', 'fish'),\n",
       " ('really', 'powerful'),\n",
       " ('long', 'one'),\n",
       " ('long', 'essay'),\n",
       " ('personal', 'project'),\n",
       " ('free', 'advertisement'),\n",
       " ('free', 'offer'),\n",
       " ('great', 'read'),\n",
       " ('political', 'thriller'),\n",
       " ('free', 'comic'),\n",
       " ('science', 'geek'),\n",
       " ('conservative', 'Christian'),\n",
       " ('short', 'introduction'),\n",
       " ('short', 'extract'),\n",
       " ('political', 'commentator'),\n",
       " ('long', 'post'),\n",
       " ('short', 'review'),\n",
       " ('political', 'philosopher'),\n",
       " ('science', 'teacher'),\n",
       " ('personal', 'article'),\n",
       " ('great', 'article'),\n",
       " ('personal', 'journal'),\n",
       " ('good', 'man'),\n",
       " ('political', 'satire'),\n",
       " ('great', 'story'),\n",
       " ('broken', 'heart'),\n",
       " ('personal', 'friend'),\n",
       " ('real', 'thing'),\n",
       " ('personal', 'interview'),\n",
       " ('conservative', 'blog'),\n",
       " ('short', 'intro'),\n",
       " ('great', 'guy'),\n",
       " ('small', 'town'),\n",
       " ('new', 'article'),\n",
       " ('political', 'analyst'),\n",
       " ('political', 'columnist'),\n",
       " ('personal', 'account'),\n",
       " ('conservative', 'writer'),\n",
       " ('dead', 'woman'),\n",
       " ('long', 'day'),\n",
       " ('recent', 'post'),\n",
       " ('truly', 'terrible'),\n",
       " ('good', 'look'),\n",
       " ('political', 'science'),\n",
       " ('short', 'tutorial'),\n",
       " ('personal', 'statement'),\n",
       " ('science', 'question'),\n",
       " ('political', 'refugee'),\n",
       " ('political', 'blog'),\n",
       " ('personal', 'preference'),\n",
       " ('great', 'question'),\n",
       " ('pretty', 'big'),\n",
       " ('good', 'point'),\n",
       " ('long', 'read'),\n",
       " ('fairly', 'interesting'),\n",
       " ('new', 'video'),\n",
       " ('good', 'one'),\n",
       " ('recent', 'article'),\n",
       " ('broken', 'record'),\n",
       " ('science', 'blog'),\n",
       " ('personal', 'favourite'),\n",
       " ('personal', 'issue'),\n",
       " ('good', 'sign'),\n",
       " ('political', 'question'),\n",
       " ('short', 'piece'),\n",
       " ('great', 'idea'),\n",
       " ('short', 'interview'),\n",
       " ('short', 'read'),\n",
       " ('dead', 'dog'),\n",
       " ('complete', 'page'),\n",
       " ('terrible', 'thing'),\n",
       " ('bad', 'movie'),\n",
       " ('complete', 'overview'),\n",
       " ('science', 'guy'),\n",
       " ('personal', 'diary'),\n",
       " ('great', 'example'),\n",
       " ('bad', 'day'),\n",
       " ('political', 'commentary'),\n",
       " ('complete', 'review'),\n",
       " ('dead', 'person'),\n",
       " ('long', 'article'),\n",
       " ('personal', 'journey'),\n",
       " ('conservative', 'thinker'),\n",
       " ('broken', 'leg'),\n",
       " ('political', 'figure'),\n",
       " ('new', 'version'),\n",
       " ('conservative', 'commentator'),\n",
       " ('dead', 'guy'),\n",
       " ('science', 'story'),\n",
       " ('good', 'read'),\n",
       " ('fairly', 'serious'),\n",
       " ('political', 'blogger'),\n",
       " ('terrible', 'idea'),\n",
       " ('personal', 'history'),\n",
       " ('short', 'answer'),\n",
       " ('personal', 'profile'),\n",
       " ('personal', 'tragedy'),\n",
       " ('political', 'conservative'),\n",
       " ('personal', 'rant'),\n",
       " ('short', 'description'),\n",
       " ('dead', 'shot'),\n",
       " ('political', 'memoir'),\n",
       " ('science', 'blogger'),\n",
       " ('conservative', 'columnist'),\n",
       " ('new', 'story'),\n",
       " ('rather', 'weird'),\n",
       " ('short', 'example'),\n",
       " ('bad', 'thing'),\n",
       " ('personal', 'decision'),\n",
       " ('good', 'guy'),\n",
       " ('personal', 'piece'),\n",
       " ('new', 'episode'),\n",
       " ('political', 'economist'),\n",
       " ('personal', 'endorsement'),\n",
       " ('good', 'story'),\n",
       " ('new', 'year'),\n",
       " ('conservative', 'website'),\n",
       " ('political', 'post'),\n",
       " ('fairly', 'scary'),\n",
       " ('short', 'speech'),\n",
       " ('great', 'book'),\n",
       " ('short', 'quiz'),\n",
       " ('political', 'party'),\n",
       " ('conservative', 'Democrat'),\n",
       " ('complete', 'chapter'),\n",
       " ('dead', 'line'),\n",
       " ('complete', 'quote'),\n",
       " ('dead', 'cert'),\n",
       " ('political', 'writer'),\n",
       " ('science', 'buff'),\n",
       " ('political', 'philosophy'),\n",
       " ('real', 'deal'),\n",
       " ('new', 'page'),\n",
       " ('pretty', 'powerful'),\n",
       " ('good', 'time'),\n",
       " ('small', 'village'),\n",
       " ('conservative', 'blogger'),\n",
       " ('science', 'student'),\n",
       " ('complete', 'recap'),\n",
       " ('new', 'chapter'),\n",
       " ('bad', 'guy'),\n",
       " ('complete', 'essay'),\n",
       " ('political', 'hack'),\n",
       " ('political', 'suicide'),\n",
       " ('complete', 'report'),\n",
       " ('dead', 'animal'),\n",
       " ('complete', 'picture'),\n",
       " ('rather', 'sad'),\n",
       " ('different', 'world'),\n",
       " ('science', 'fan'),\n",
       " ('conservative', 'conservative'),\n",
       " ('pretty', 'crazy'),\n",
       " ('broken', 'man'),\n",
       " ('complete', 'guide'),\n",
       " ('good', 'book'),\n",
       " ('science', 'experiment'),\n",
       " ('new', 'feature'),\n",
       " ('science', 'magazine'),\n",
       " ('conservative', 'journalist'),\n",
       " ('bad', 'joke'),\n",
       " ('political', 'document'),\n",
       " ('science', 'perspective'),\n",
       " ('new', 'series'),\n",
       " ('complete', 'explanation'),\n",
       " ('recent', 'addition'),\n",
       " ('complete', 'excerpt'),\n",
       " ('bad', 'sign'),\n",
       " ('great', 'writer'),\n",
       " ('rather', 'bizarre'),\n",
       " ('dead', 'joke'),\n",
       " ('conservative', 'magazine'),\n",
       " ('conservative', 'perspective'),\n",
       " ('science', 'man'),\n",
       " ('complete', 'description'),\n",
       " ('new', 'day'),\n",
       " ('political', 'phenomenon'),\n",
       " ('great', 'post'),\n",
       " ('conservative', 'article'),\n",
       " ('recent', 'video'),\n",
       " ('science', 'book'),\n",
       " ('dead', 'duck'),\n",
       " ('rather', 'scary'),\n",
       " ('dead', 'sentence'),\n",
       " ('science', 'nut'),\n",
       " ('great', 'quote'),\n",
       " ('great', 'pick'),\n",
       " ('complete', 'reconstruction'),\n",
       " ('science', 'columnist'),\n",
       " ('new', 'site'),\n",
       " ('new', 'entry'),\n",
       " ('recent', 'interview'),\n",
       " ('good', 'argument'),\n",
       " ('dead', 'one'),\n",
       " ('terrible', 'movie'),\n",
       " ('conservative', 'story'),\n",
       " ('great', 'interview'),\n",
       " ('terrible', 'man'),\n",
       " ('real', 'fucking'),\n",
       " ('great', 'list'),\n",
       " ('completely', 'different'),\n",
       " ('rather', 'bad'),\n",
       " ('dead', 'letter'),\n",
       " ('complete', 'mess'),\n",
       " ('terrible', 'story'),\n",
       " ('dead', 'dragon'),\n",
       " ('dead', 'sheep'),\n",
       " ('great', 'song'),\n",
       " ('bad', 'analogy'),\n",
       " ('science', 'article'),\n",
       " ('bad', 'example'),\n",
       " ('conservative', 'Catholic'),\n",
       " ('dead', 'beat'),\n",
       " ('conservative', 'site'),\n",
       " ('good', 'analogy'),\n",
       " ('dead', 'version'),\n",
       " ('recent', 'update'),\n",
       " ('rather', 'disturbing'),\n",
       " ('complete', 'FAQ'),\n",
       " ('long', 'year'),\n",
       " ('good', 'list'),\n",
       " ('science', 'editor'),\n",
       " ('great', 'time'),\n",
       " ('great', 'start'),\n",
       " ('real', 'problem'),\n",
       " ('dead', 'show'),\n",
       " ('conservative', 'American'),\n",
       " ('complete', 'synopsis'),\n",
       " ('dead', 'game'),\n",
       " ('dead', 'baby'),\n",
       " ('recent', 'episode'),\n",
       " ('recent', 'excerpt'),\n",
       " ('conservative', 'book'),\n",
       " ('great', 'man'),\n",
       " ('recent', 'story'),\n",
       " ('conservative', 'economist'),\n",
       " ('good', 'pick'),\n",
       " ('conservative', 'post'),\n",
       " ('real', 'story'),\n",
       " ('science', 'graduate'),\n",
       " ('small', 'house'),\n",
       " ('great', 'point'),\n",
       " ('science', 'critic'),\n",
       " ('dead', 'thread'),\n",
       " ('complete', 'video'),\n",
       " ('terrible', 'person'),\n",
       " ('different', 'animal'),\n",
       " ('great', 'game'),\n",
       " ('great', 'video'),\n",
       " ('science', 'major'),\n",
       " ('good', 'job'),\n",
       " ('small', 'world'),\n",
       " ('complete', 'statement'),\n",
       " ('terrible', 'guy'),\n",
       " ('conservative', 'favorite'),\n",
       " ('real', 'shame'),\n",
       " ('long', 'journey'),\n",
       " ('great', 'player'),\n",
       " ('good', 'answer'),\n",
       " ('recent', 'reprint'),\n",
       " ('science', 'kid'),\n",
       " ('good', 'call'),\n",
       " ('broken', 'bone'),\n",
       " ('good', 'plan'),\n",
       " ('great', 'movie'),\n",
       " ('real', 'live'),\n",
       " ('conservative', 'icon'),\n",
       " ('science', 'reader'),\n",
       " ('good', 'article'),\n",
       " ('bad', 'book'),\n",
       " ('bad', 'man'),\n",
       " ('good', 'game'),\n",
       " ('complete', 'spoiler'),\n",
       " ('complete', 'rundown'),\n",
       " ('recent', 'photo'),\n",
       " ('science', 'degree'),\n",
       " ('conservative', 'reporter'),\n",
       " ('conservative', 'newspaper'),\n",
       " ('complete', 'collection'),\n",
       " ('new', 'website'),\n",
       " ('broken', 'window'),\n",
       " ('small', 'farm'),\n",
       " ('good', 'choice'),\n",
       " ('great', 'analogy'),\n",
       " ('great', 'compliment'),\n",
       " ('conservative', 'libertarian'),\n",
       " ('good', 'name'),\n",
       " ('great', 'job'),\n",
       " ('great', 'philosopher'),\n",
       " ('long', 'book'),\n",
       " ('new', 'one'),\n",
       " ('broken', 'arm'),\n",
       " ('good', 'shot'),\n",
       " ('terrible', 'example'),\n",
       " ('broken', 'mirror'),\n",
       " ('recent', 'illustration'),\n",
       " ('recent', 'study'),\n",
       " ('new', 'game'),\n",
       " ('terrible', 'mistake'),\n",
       " ('terrible', 'shame'),\n",
       " ('terrible', 'book'),\n",
       " ('real', 'person'),\n",
       " ('recent', 'reference'),\n",
       " ('new', 'blog'),\n",
       " ('little', 'scary'),\n",
       " ('long', 'way'),\n",
       " ('recent', 'entry'),\n",
       " ('conservative', 'column'),\n",
       " ('conservative', 'organization'),\n",
       " ('terrible', 'day'),\n",
       " ('long', 'weekend'),\n",
       " ('new', 'bie'),\n",
       " ('good', 'line'),\n",
       " ('long', 'answer'),\n",
       " ('bad', 'time'),\n",
       " ('long', 'rant'),\n",
       " ('bad', 'year'),\n",
       " ('great', 'name'),\n",
       " ('bad', 'start'),\n",
       " ('long', 'process'),\n",
       " ('recent', 'image'),\n",
       " ('very', 'real'),\n",
       " ('conservative', 'movement'),\n",
       " ('long', 'pause'),\n",
       " ('recent', 'issue'),\n",
       " ('bad', 'choice'),\n",
       " ('new', 'addition'),\n",
       " ('bad', 'picture'),\n",
       " ('new', 'book'),\n",
       " ('new', 'movie'),\n",
       " ('great', 'review'),\n",
       " ('long', 'wait'),\n",
       " ('recent', 'review'),\n",
       " ('bad', 'story'),\n",
       " ('great', 'introduction'),\n",
       " ('bad', 'song'),\n",
       " ('terrible', 'joke'),\n",
       " ('small', 'island'),\n",
       " ('long', 'paragraph'),\n",
       " ('new', 'era'),\n",
       " ('different', 'question'),\n",
       " ('long', 'trip'),\n",
       " ('different', 'beast'),\n",
       " ('new', 'interview'),\n",
       " ('long', 'slog'),\n",
       " ('little', 'blurry'),\n",
       " ('recent', 'one'),\n",
       " ('long', 'season'),\n",
       " ('new', 'format'),\n",
       " ('broken', 'glass'),\n",
       " ('terrible', 'film'),\n",
       " ('terrible', 'feeling'),\n",
       " ('little', 'different'),\n",
       " ('different', 'situation'),\n",
       " ('recent', 'translation'),\n",
       " ('broken', 'person'),\n",
       " ('long', 'life'),\n",
       " ('long', 'week'),\n",
       " ('terrible', 'picture'),\n",
       " ('broken', 'computer'),\n",
       " ('different', 'game'),\n",
       " ('recent', 'transcript'),\n",
       " ('long', 'letter'),\n",
       " ('terrible', 'time'),\n",
       " ('broken', 'link'),\n",
       " ('very', 'difficult'),\n",
       " ('long', 'haul'),\n",
       " ('different', 'picture'),\n",
       " ('little', 'better'),\n",
       " ('terrible', 'analogy'),\n",
       " ('terrible', 'song'),\n",
       " ('real', 'man'),\n",
       " ('long', 'night'),\n",
       " ('real', 'question'),\n",
       " ('terrible', 'sentence'),\n",
       " ('long', 'while'),\n",
       " ('recent', 'case'),\n",
       " ('different', 'language'),\n",
       " ('recent', 'essay'),\n",
       " ('new', 'thread'),\n",
       " ('recent', 'version'),\n",
       " ('broken', 'body'),\n",
       " ('bad', 'game'),\n",
       " ('truly', 'lovely'),\n",
       " ('bad', 'person'),\n",
       " ('bad', 'film'),\n",
       " ('recent', 'quote'),\n",
       " ('bad', 'word'),\n",
       " ('terrible', 'thought'),\n",
       " ('broken', 'dog'),\n",
       " ('broken', 'neck'),\n",
       " ('recent', 'event'),\n",
       " ('real', 'joke'),\n",
       " ('bad', 'metaphor'),\n",
       " ('different', 'person'),\n",
       " ('bad', 'boy'),\n",
       " ('new', 'breed'),\n",
       " ('real', 'possibility'),\n",
       " ('recent', 'release'),\n",
       " ('little', 'more'),\n",
       " ('new', 'logo'),\n",
       " ('terrible', 'writer'),\n",
       " ('real', 'difference'),\n",
       " ('broken', 'hand'),\n",
       " ('different', 'man'),\n",
       " ('broken', 'limb'),\n",
       " ('small', 'city'),\n",
       " ('new', 'design'),\n",
       " ('real', 'guy'),\n",
       " ('bad', 'memory'),\n",
       " ('new', 'age'),\n",
       " ('bad', 'feeling'),\n",
       " ('broken', 'cell'),\n",
       " ('broken', 'metaphor'),\n",
       " ('bad', 'kid'),\n",
       " ('broken', 'dream'),\n",
       " ('small', 'country'),\n",
       " ('very', 'unusual'),\n",
       " ('terrible', 'tragedy'),\n",
       " ('different', 'day'),\n",
       " ('terrible', 'word'),\n",
       " ('terrible', 'record'),\n",
       " ('recent', 'sample'),\n",
       " ('bad', 'decision'),\n",
       " ('real', 'ist'),\n",
       " ('terrible', 'sign'),\n",
       " ('bad', 'pun'),\n",
       " ('recent', 'development'),\n",
       " ('different', 'thing'),\n",
       " ('totally', 'different'),\n",
       " ('broken', 'nose'),\n",
       " ('real', 'mess'),\n",
       " ('bad', 'dream'),\n",
       " ('terrible', 'loss'),\n",
       " ('little', 'sad'),\n",
       " ('different', 'country'),\n",
       " ('different', 'view'),\n",
       " ('different', 'time'),\n",
       " ('terrible', 'name'),\n",
       " ('different', 'perspective'),\n",
       " ('little', 'longer'),\n",
       " ('terrible', 'year'),\n",
       " ('broken', 'car'),\n",
       " ('small', 'garden'),\n",
       " ('different', 'matter'),\n",
       " ('terrible', 'place'),\n",
       " ('completely', 'new'),\n",
       " ('broken', 'machine'),\n",
       " ('small', 'room'),\n",
       " ('different', 'problem'),\n",
       " ('real', 'job'),\n",
       " ('small', 'family'),\n",
       " ('small', 'child'),\n",
       " ('small', 'community'),\n",
       " ('real', 'pain'),\n",
       " ('small', 'school'),\n",
       " ('real', 'real'),\n",
       " ('broken', 'phone'),\n",
       " ('broken', 'ankle'),\n",
       " ('broken', 'sign'),\n",
       " ('broken', 'rib'),\n",
       " ('small', 'boat'),\n",
       " ('little', 'rough'),\n",
       " ('broken', 'woman'),\n",
       " ('small', 'dog'),\n",
       " ('little', 'easier'),\n",
       " ('broken', 'cat'),\n",
       " ('little', 'confusing'),\n",
       " ('different', 'life'),\n",
       " ('real', 'tragedy'),\n",
       " ('different', 'approach'),\n",
       " ('broken', 'spine'),\n",
       " ('different', 'book'),\n",
       " ('small', 'boy'),\n",
       " ('real', 'woman'),\n",
       " ('small', 'planet'),\n",
       " ('little', 'closer'),\n",
       " ('different', 'experience'),\n",
       " ('rather', 'peculiar'),\n",
       " ('little', 'embarrassing'),\n",
       " ('different', 'planet'),\n",
       " ('small', 'business'),\n",
       " ('different', 'dog'),\n",
       " ('real', 'threat'),\n",
       " ('real', 'estate'),\n",
       " ('real', 'bitch'),\n",
       " ('real', 'issue'),\n",
       " ('small', 'apartment'),\n",
       " ('small', 'tree'),\n",
       " ('different', 'idea'),\n",
       " ('little', 'disappointing'),\n",
       " ('small', 'life'),\n",
       " ('small', 'space'),\n",
       " ('little', 'disturbing'),\n",
       " ('different', 'era'),\n",
       " ('small', 'building'),\n",
       " ('little', 'tricky'),\n",
       " ('small', 'man'),\n",
       " ('different', 'culture'),\n",
       " ('little', 'complicated'),\n",
       " ('real', 'crisis'),\n",
       " ('small', 'area'),\n",
       " ('different', 'breed'),\n",
       " ('little', 'harder'),\n",
       " ('real', 'challenge'),\n",
       " ('little', 'worried'),\n",
       " ('small', 'church'),\n",
       " ('real', 'killer'),\n",
       " ('different', 'guy'),\n",
       " ('real', 'nice'),\n",
       " ('small', 'place'),\n",
       " ('small', 'girl'),\n",
       " ('real', 'word'),\n",
       " ('small', 'barn'),\n",
       " ('little', 'creepy'),\n",
       " ('little', 'later'),\n",
       " ('little', 'bit'),\n",
       " ('totally', 'normal'),\n",
       " ('rather', 'nasty'),\n",
       " ('totally', 'new'),\n",
       " ('little', 'clearer'),\n",
       " ('completely', 'normal'),\n",
       " ('pretty', 'serious'),\n",
       " ('little', 'misleading'),\n",
       " ('little', 'weird'),\n",
       " ('little', 'messy'),\n",
       " ('little', 'fuzzy'),\n",
       " ('real', 'funny'),\n",
       " ('little', 'frightening'),\n",
       " ('fairly', 'common'),\n",
       " ('little', 'unsettling'),\n",
       " ('little', 'difficult'),\n",
       " ('little', 'simpler'),\n",
       " ('totally', 'awesome'),\n",
       " ('totally', 'amazing'),\n",
       " ('totally', 'random'),\n",
       " ('totally', 'good'),\n",
       " ('completely', 'amazing'),\n",
       " ('truly', 'crazy'),\n",
       " ('fairly', 'rare'),\n",
       " ('completely', 'real'),\n",
       " ('totally', 'real'),\n",
       " ('totally', 'weird'),\n",
       " ('totally', 'bad'),\n",
       " ('completely', 'insane'),\n",
       " ('totally', 'crazy'),\n",
       " ('truly', 'horrible'),\n",
       " ('very', 'cool'),\n",
       " ('totally', 'insane'),\n",
       " ('completely', 'crazy'),\n",
       " ('fairly', 'normal'),\n",
       " ('completely', 'random'),\n",
       " ('fairly', 'complex'),\n",
       " ('completely', 'bizarre'),\n",
       " ('completely', 'terrible'),\n",
       " ('totally', 'terrible'),\n",
       " ('completely', 'horrible'),\n",
       " ('very', 'weird'),\n",
       " ('completely', 'unique'),\n",
       " ('fairly', 'complicated'),\n",
       " ('really', 'fun'),\n",
       " ('completely', 'bad'),\n",
       " ('fairly', 'powerful'),\n",
       " ('fairly', 'large'),\n",
       " ('real', 'human'),\n",
       " ('very', 'simple'),\n",
       " ('totally', 'stupid'),\n",
       " ('really', 'neat'),\n",
       " ('totally', 'bizarre'),\n",
       " ('very', 'different'),\n",
       " ('totally', 'ridiculous'),\n",
       " ('more', 'complicated'),\n",
       " ('really', 'stupid'),\n",
       " ('rather', 'complicated'),\n",
       " ('more', 'complex'),\n",
       " ('pretty', 'neat'),\n",
       " ('more', 'interesting'),\n",
       " ('really', 'dumb'),\n",
       " ('real', 'living'),\n",
       " ('really', 'sweet'),\n",
       " ('more', 'serious'),\n",
       " ('great', 'little'),\n",
       " ('real', 'interesting'),\n",
       " ('more', 'powerful'),\n",
       " ('really', 'strong'),\n",
       " ('pretty', 'fun'),\n",
       " ('more', 'normal'),\n",
       " ('good', 'little'),\n",
       " ('real', 'ugly'),\n",
       " ('fairly', 'simple'),\n",
       " ('good', 'old'),\n",
       " ('more', 'real'),\n",
       " ('fairly', 'good'),\n",
       " ('more', 'nuanced'),\n",
       " ('more', 'human'),\n",
       " ('more', 'positive'),\n",
       " ('fairly', 'small'),\n",
       " ('great', 'American'),\n",
       " ('truly', 'exceptional'),\n",
       " ('fucking', 'good'),\n",
       " ('pretty', 'great'),\n",
       " ('truly', 'unique'),\n",
       " ('truly', 'scary'),\n",
       " ('fairly', 'straightforward'),\n",
       " ('really', 'big'),\n",
       " ('truly', 'magical'),\n",
       " ('fairly', 'standard'),\n",
       " ('great', 'good'),\n",
       " ('fucking', 'fucking'),\n",
       " ('truly', 'brilliant'),\n",
       " ('good', 'good'),\n",
       " ('rather', 'good'),\n",
       " ('super', 'weird'),\n",
       " ('fairly', 'basic'),\n",
       " ('good', 'American'),\n",
       " ('super', 'awesome'),\n",
       " ('truly', 'astonishing'),\n",
       " ('truly', 'sad'),\n",
       " ('super', 'good'),\n",
       " ('great', 'great'),\n",
       " ('fucking', 'bad'),\n",
       " ('super', 'bad'),\n",
       " ('fucking', 'terrible'),\n",
       " ('super', 'fun'),\n",
       " ('fucking', 'great'),\n",
       " ('fucking', 'shit'),\n",
       " ('super', 'powerful'),\n",
       " ('fucking', 'stupid'),\n",
       " ('very', 'violent'),\n",
       " ('rather', 'ugly'),\n",
       " ('fucking', 'shitty'),\n",
       " ('rather', 'dark'),\n",
       " ('fucking', 'weird'),\n",
       " ('fucking', 'horrible'),\n",
       " ('fucking', 'funny'),\n",
       " ('super', 'nice'),\n",
       " ('fucking', 'scary'),\n",
       " ('pretty', 'impressive'),\n",
       " ('really', 'cute'),\n",
       " ('super', 'scary'),\n",
       " ('super', 'stupid'),\n",
       " ('good', 'fucking'),\n",
       " ('super', 'exciting'),\n",
       " ('highly', 'intelligent'),\n",
       " ('pretty', 'remarkable'),\n",
       " ('super', 'interesting'),\n",
       " ('super', 'dumb'),\n",
       " ('super', 'cute'),\n",
       " ('super', 'funny'),\n",
       " ('great', 'new'),\n",
       " ('fucking', 'goddamn'),\n",
       " ('great', 'fucking'),\n",
       " ('typical', 'American'),\n",
       " ('great', 'first'),\n",
       " ('good', 'political'),\n",
       " ('pretty', 'decent'),\n",
       " ('highly', 'emotional'),\n",
       " ('pretty', 'boring'),\n",
       " ('highly', 'successful'),\n",
       " ('typical', 'old'),\n",
       " ('truly', 'awful'),\n",
       " ('highly', 'controversial'),\n",
       " ('pretty', 'terrible'),\n",
       " ('highly', 'unusual'),\n",
       " ('pretty', 'awful'),\n",
       " ('fairly', 'violent'),\n",
       " ('fairly', 'boring'),\n",
       " ('rather', 'big'),\n",
       " ('fairly', 'decent'),\n",
       " ('very', 'big'),\n",
       " ('highly', 'interesting'),\n",
       " ('rather', 'small'),\n",
       " ('rather', 'remarkable'),\n",
       " ('highly', 'influential'),\n",
       " ('very', 'rare'),\n",
       " ('fairly', 'bad'),\n",
       " ('fairly', 'obscure'),\n",
       " ('fairly', 'dark'),\n",
       " ('fairly', 'typical'),\n",
       " ('very', 'small'),\n",
       " ('rather', 'boring'),\n",
       " ('very', 'positive'),\n",
       " ('relatively', 'new'),\n",
       " ('very', 'smart'),\n",
       " ('very', 'exciting'),\n",
       " ('very', 'intelligent'),\n",
       " ('whole', 'other'),\n",
       " ('big', 'scary'),\n",
       " ('hugely', 'important'),\n",
       " ('big', 'bad'),\n",
       " ('really', 'smart'),\n",
       " ('very', 'common'),\n",
       " ('really', 'small'),\n",
       " ('pretty', 'common'),\n",
       " ('perfectly', 'normal'),\n",
       " ('pretty', 'important'),\n",
       " ('very', 'significant'),\n",
       " ('very', 'brave'),\n",
       " ('very', 'strong'),\n",
       " ('really', 'exciting'),\n",
       " ('very', 'sick'),\n",
       " ('very', 'valuable'),\n",
       " ('really', 'rare'),\n",
       " ('whole', 'different'),\n",
       " ('very', 'angry'),\n",
       " ('really', 'huge'),\n",
       " ('really', 'crazy'),\n",
       " ('pretty', 'rare'),\n",
       " ('very', 'talented'),\n",
       " ('big', 'big'),\n",
       " ('really', 'hard'),\n",
       " ('relatively', 'small'),\n",
       " ('pretty', 'simple'),\n",
       " ('whole', 'new'),\n",
       " ('big', 'fat'),\n",
       " ('relatively', 'recent'),\n",
       " ('pretty', 'huge'),\n",
       " ('really', 'dangerous'),\n",
       " ('big', 'old'),\n",
       " ('pretty', 'significant'),\n",
       " ('pretty', 'small'),\n",
       " ('pretty', 'exciting'),\n",
       " ('pretty', 'incredible'),\n",
       " ('fairly', 'new'),\n",
       " ('seriously', 'sick'),\n",
       " ('seriously', 'disturbed'),\n",
       " ('pretty', 'beautiful'),\n",
       " ('relatively', 'common'),\n",
       " ('perfectly', 'sane'),\n",
       " ('relatively', 'rare'),\n",
       " ('really', 'special'),\n",
       " ('seriously', 'bad'),\n",
       " ('perfectly', 'decent'),\n",
       " ('good', 'luck'),\n",
       " ('fairly', 'big'),\n",
       " ('hugely', 'positive'),\n",
       " ('really', 'lovely'),\n",
       " ('truly', 'miraculous'),\n",
       " ('perfectly', 'good'),\n",
       " ('good', 'moral'),\n",
       " ('seriously', 'ill'),\n",
       " ('really', 'sick'),\n",
       " ('relatively', 'simple'),\n",
       " ('seriously', 'flawed'),\n",
       " ('great', 'big'),\n",
       " ('hugely', 'exciting'),\n",
       " ('perfectly', 'reasonable'),\n",
       " ('hugely', 'powerful'),\n",
       " ('perfectly', 'fine'),\n",
       " ('seriously', 'dangerous'),\n",
       " ('big', 'beautiful'),\n",
       " ('relatively', 'minor'),\n",
       " ('perfectly', 'nice'),\n",
       " ('hugely', 'significant'),\n",
       " ('fairly', 'recent'),\n",
       " ('good', 'business'),\n",
       " ('good', 'news'),\n",
       " ('seriously', 'insane'),\n",
       " ('totally', 'cool'),\n",
       " ('seriously', 'scary'),\n",
       " ('pretty', 'smart'),\n",
       " ('fairly', 'routine'),\n",
       " ('relatively', 'big'),\n",
       " ('relatively', 'routine'),\n",
       " ('perfectly', 'happy'),\n",
       " ('truly', 'inspiring'),\n",
       " ('truly', 'marvelous'),\n",
       " ('hugely', 'good'),\n",
       " ('relatively', 'trivial'),\n",
       " ('relatively', 'old'),\n",
       " ('relatively', 'easy'),\n",
       " ('typical', 'homeless'),\n",
       " ('completely', 'good'),\n",
       " ('perfectly', 'rational'),\n",
       " ('truly', 'rare'),\n",
       " ('hugely', 'scary'),\n",
       " ('seriously', 'crazy'),\n",
       " ('fairly', 'trivial'),\n",
       " ('truly', 'important'),\n",
       " ('fairly', 'important'),\n",
       " ('seriously', 'evil'),\n",
       " ('perfectly', 'ordinary'),\n",
       " ('hugely', 'valuable'),\n",
       " ('truly', 'gifted'),\n",
       " ('seriously', 'stupid'),\n",
       " ('perfectly', 'healthy'),\n",
       " ('rather', 'rare'),\n",
       " ('fairly', 'significant'),\n",
       " ('fucking', 'amazing'),\n",
       " ('truly', 'fine'),\n",
       " ('seriously', 'damaged'),\n",
       " ('completely', 'unrelated'),\n",
       " ('completely', 'separate'),\n",
       " ('truly', 'terrifying'),\n",
       " ('truly', 'admirable'),\n",
       " ('super', 'important'),\n",
       " ('fucking', 'real'),\n",
       " ('truly', 'frightening'),\n",
       " ('fucking', 'beautiful'),\n",
       " ('real', 'black'),\n",
       " ('perfectly', 'imperfect'),\n",
       " ('fairly', 'obvious'),\n",
       " ('hugely', 'negative'),\n",
       " ('pretty', 'tough'),\n",
       " ('totally', 'beautiful'),\n",
       " ('pretty', 'badass'),\n",
       " ('fairly', 'minor'),\n",
       " ('seriously', 'strange'),\n",
       " ('completely', 'weird'),\n",
       " ('hugely', 'useful'),\n",
       " ('fairly', 'regular'),\n",
       " ('truly', 'strange'),\n",
       " ('hugely', 'interesting'),\n",
       " ('rather', 'important'),\n",
       " ('relatively', 'young'),\n",
       " ('fairly', 'commonplace'),\n",
       " ('completely', 'unexpected'),\n",
       " ('relatively', 'good'),\n",
       " ('hugely', 'big'),\n",
       " ('completely', 'wonderful'),\n",
       " ('relatively', 'normal'),\n",
       " ('perfectly', 'human'),\n",
       " ('relatively', 'unknown'),\n",
       " ('pretty', 'special'),\n",
       " ('fairly', 'cool'),\n",
       " ('relatively', 'straightforward'),\n",
       " ('seriously', 'troubled'),\n",
       " ('completely', 'awesome'),\n",
       " ('fairly', 'remarkable'),\n",
       " ('totally', 'human'),\n",
       " ('hugely', 'dangerous'),\n",
       " ...]"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.suggest(('This is a {bert} {bert} {t}.', 'John is a {bert} {bert}.'), t=['thing', 'movie', 'person', 'man'], verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'23 John is a 23 man'"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'\\d+$', '', '23 John is a 23 man231')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'male': ['Michael',\n",
       "  'Christopher',\n",
       "  'Matthew',\n",
       "  'David',\n",
       "  'James',\n",
       "  'John',\n",
       "  'Joshua',\n",
       "  'Daniel',\n",
       "  'Joseph',\n",
       "  'William',\n",
       "  'Robert',\n",
       "  'Andrew',\n",
       "  'Jason',\n",
       "  'Ryan',\n",
       "  'Anthony',\n",
       "  'Jacob',\n",
       "  'Nicholas',\n",
       "  'Brian',\n",
       "  'Justin',\n",
       "  'Jonathan',\n",
       "  'Brandon',\n",
       "  'Kevin',\n",
       "  'Thomas',\n",
       "  'Eric',\n",
       "  'Benjamin',\n",
       "  'Alexander',\n",
       "  'Tyler',\n",
       "  'Steven',\n",
       "  'Timothy',\n",
       "  'Zachary',\n",
       "  'Charles',\n",
       "  'Richard',\n",
       "  'Aaron',\n",
       "  'Adam',\n",
       "  'Jordan',\n",
       "  'Nathan',\n",
       "  'Samuel',\n",
       "  'Kyle',\n",
       "  'Mark',\n",
       "  'Jeffrey',\n",
       "  'Jose',\n",
       "  'Jeremy',\n",
       "  'Ethan',\n",
       "  'Christian',\n",
       "  'Austin',\n",
       "  'Noah',\n",
       "  'Dylan',\n",
       "  'Scott',\n",
       "  'Sean',\n",
       "  'Patrick',\n",
       "  'Logan',\n",
       "  'Paul',\n",
       "  'Stephen',\n",
       "  'Gabriel',\n",
       "  'Kenneth',\n",
       "  'Angel',\n",
       "  'Bryan',\n",
       "  'Cameron',\n",
       "  'Gregory',\n",
       "  'Cody',\n",
       "  'Caleb',\n",
       "  'Jesse',\n",
       "  'Elijah',\n",
       "  'Mason',\n",
       "  'Juan',\n",
       "  'Travis',\n",
       "  'Shawn',\n",
       "  'Luke',\n",
       "  'Evan',\n",
       "  'Lucas',\n",
       "  'Isaac',\n",
       "  'Edward',\n",
       "  'Jack',\n",
       "  'Hunter',\n",
       "  'Luis',\n",
       "  'Jayden',\n",
       "  'Carlos',\n",
       "  'Jackson',\n",
       "  'Adrian',\n",
       "  'Alex',\n",
       "  'Chad',\n",
       "  'Bradley',\n",
       "  'Ian',\n",
       "  'Nathaniel',\n",
       "  'Liam',\n",
       "  'Connor',\n",
       "  'Isaiah',\n",
       "  'Aiden',\n",
       "  'Derek',\n",
       "  'Peter',\n",
       "  'Dustin',\n",
       "  'George',\n",
       "  'Jared',\n",
       "  'Marcus',\n",
       "  'Antonio',\n",
       "  'Henry',\n",
       "  'Jesus',\n",
       "  'Jeremiah',\n",
       "  'Julian',\n",
       "  'Donald'],\n",
       " 'female': ['Jennifer',\n",
       "  'Jessica',\n",
       "  'Ashley',\n",
       "  'Sarah',\n",
       "  'Emily',\n",
       "  'Amanda',\n",
       "  'Elizabeth',\n",
       "  'Melissa',\n",
       "  'Stephanie',\n",
       "  'Nicole',\n",
       "  'Samantha',\n",
       "  'Michelle',\n",
       "  'Kimberly',\n",
       "  'Amy',\n",
       "  'Heather',\n",
       "  'Rachel',\n",
       "  'Lauren',\n",
       "  'Rebecca',\n",
       "  'Angela',\n",
       "  'Emma',\n",
       "  'Megan',\n",
       "  'Taylor',\n",
       "  'Olivia',\n",
       "  'Hannah',\n",
       "  'Alexis',\n",
       "  'Madison',\n",
       "  'Christina',\n",
       "  'Lisa',\n",
       "  'Mary',\n",
       "  'Amber',\n",
       "  'Kelly',\n",
       "  'Abigail',\n",
       "  'Brittany',\n",
       "  'Laura',\n",
       "  'Danielle',\n",
       "  'Victoria',\n",
       "  'Katherine',\n",
       "  'Kayla',\n",
       "  'Tiffany',\n",
       "  'Andrea',\n",
       "  'Sophia',\n",
       "  'Isabella',\n",
       "  'Anna',\n",
       "  'Natalie',\n",
       "  'Alyssa',\n",
       "  'Erin',\n",
       "  'Jamie',\n",
       "  'Maria',\n",
       "  'Sara',\n",
       "  'Shannon',\n",
       "  'Crystal',\n",
       "  'Allison',\n",
       "  'Courtney',\n",
       "  'Brianna',\n",
       "  'Ava',\n",
       "  'Jasmine',\n",
       "  'Morgan',\n",
       "  'Grace',\n",
       "  'Julie',\n",
       "  'Alexandra',\n",
       "  'Mia',\n",
       "  'Erica',\n",
       "  'Julia',\n",
       "  'Christine',\n",
       "  'Vanessa',\n",
       "  'Chloe',\n",
       "  'Kristen',\n",
       "  'April',\n",
       "  'Riley',\n",
       "  'Brooke',\n",
       "  'Leah',\n",
       "  'Melanie',\n",
       "  'Karen',\n",
       "  'Alicia',\n",
       "  'Kathryn',\n",
       "  'Katie',\n",
       "  'Jacqueline',\n",
       "  'Cynthia',\n",
       "  'Avery',\n",
       "  'Monica',\n",
       "  'Catherine',\n",
       "  'Sydney',\n",
       "  'Savannah',\n",
       "  'Leslie',\n",
       "  'Patricia',\n",
       "  'Kaitlyn',\n",
       "  'Hailey',\n",
       "  'Chelsea',\n",
       "  'Lindsey',\n",
       "  'Tara',\n",
       "  'Haley',\n",
       "  'Tracy',\n",
       "  'Evelyn',\n",
       "  'Ella',\n",
       "  'Kelsey',\n",
       "  'Kristin',\n",
       "  'Destiny',\n",
       "  'Diana',\n",
       "  'Holly',\n",
       "  'Caroline'],\n",
       " 'male_pron': [Munch({'name': 'Michael', 'pron': 'he'}),\n",
       "  Munch({'name': 'Christopher', 'pron': 'he'}),\n",
       "  Munch({'name': 'Matthew', 'pron': 'he'}),\n",
       "  Munch({'name': 'David', 'pron': 'he'}),\n",
       "  Munch({'name': 'James', 'pron': 'he'}),\n",
       "  Munch({'name': 'John', 'pron': 'he'}),\n",
       "  Munch({'name': 'Joshua', 'pron': 'he'}),\n",
       "  Munch({'name': 'Daniel', 'pron': 'he'}),\n",
       "  Munch({'name': 'Joseph', 'pron': 'he'}),\n",
       "  Munch({'name': 'William', 'pron': 'he'}),\n",
       "  Munch({'name': 'Robert', 'pron': 'he'}),\n",
       "  Munch({'name': 'Andrew', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jason', 'pron': 'he'}),\n",
       "  Munch({'name': 'Ryan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Anthony', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jacob', 'pron': 'he'}),\n",
       "  Munch({'name': 'Nicholas', 'pron': 'he'}),\n",
       "  Munch({'name': 'Brian', 'pron': 'he'}),\n",
       "  Munch({'name': 'Justin', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jonathan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Brandon', 'pron': 'he'}),\n",
       "  Munch({'name': 'Kevin', 'pron': 'he'}),\n",
       "  Munch({'name': 'Thomas', 'pron': 'he'}),\n",
       "  Munch({'name': 'Eric', 'pron': 'he'}),\n",
       "  Munch({'name': 'Benjamin', 'pron': 'he'}),\n",
       "  Munch({'name': 'Alexander', 'pron': 'he'}),\n",
       "  Munch({'name': 'Tyler', 'pron': 'he'}),\n",
       "  Munch({'name': 'Steven', 'pron': 'he'}),\n",
       "  Munch({'name': 'Timothy', 'pron': 'he'}),\n",
       "  Munch({'name': 'Zachary', 'pron': 'he'}),\n",
       "  Munch({'name': 'Charles', 'pron': 'he'}),\n",
       "  Munch({'name': 'Richard', 'pron': 'he'}),\n",
       "  Munch({'name': 'Aaron', 'pron': 'he'}),\n",
       "  Munch({'name': 'Adam', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jordan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Nathan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Samuel', 'pron': 'he'}),\n",
       "  Munch({'name': 'Kyle', 'pron': 'he'}),\n",
       "  Munch({'name': 'Mark', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jeffrey', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jose', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jeremy', 'pron': 'he'}),\n",
       "  Munch({'name': 'Ethan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Christian', 'pron': 'he'}),\n",
       "  Munch({'name': 'Austin', 'pron': 'he'}),\n",
       "  Munch({'name': 'Noah', 'pron': 'he'}),\n",
       "  Munch({'name': 'Dylan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Scott', 'pron': 'he'}),\n",
       "  Munch({'name': 'Sean', 'pron': 'he'}),\n",
       "  Munch({'name': 'Patrick', 'pron': 'he'}),\n",
       "  Munch({'name': 'Logan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Paul', 'pron': 'he'}),\n",
       "  Munch({'name': 'Stephen', 'pron': 'he'}),\n",
       "  Munch({'name': 'Gabriel', 'pron': 'he'}),\n",
       "  Munch({'name': 'Kenneth', 'pron': 'he'}),\n",
       "  Munch({'name': 'Angel', 'pron': 'he'}),\n",
       "  Munch({'name': 'Bryan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Cameron', 'pron': 'he'}),\n",
       "  Munch({'name': 'Gregory', 'pron': 'he'}),\n",
       "  Munch({'name': 'Cody', 'pron': 'he'}),\n",
       "  Munch({'name': 'Caleb', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jesse', 'pron': 'he'}),\n",
       "  Munch({'name': 'Elijah', 'pron': 'he'}),\n",
       "  Munch({'name': 'Mason', 'pron': 'he'}),\n",
       "  Munch({'name': 'Juan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Travis', 'pron': 'he'}),\n",
       "  Munch({'name': 'Shawn', 'pron': 'he'}),\n",
       "  Munch({'name': 'Luke', 'pron': 'he'}),\n",
       "  Munch({'name': 'Evan', 'pron': 'he'}),\n",
       "  Munch({'name': 'Lucas', 'pron': 'he'}),\n",
       "  Munch({'name': 'Isaac', 'pron': 'he'}),\n",
       "  Munch({'name': 'Edward', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jack', 'pron': 'he'}),\n",
       "  Munch({'name': 'Hunter', 'pron': 'he'}),\n",
       "  Munch({'name': 'Luis', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jayden', 'pron': 'he'}),\n",
       "  Munch({'name': 'Carlos', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jackson', 'pron': 'he'}),\n",
       "  Munch({'name': 'Adrian', 'pron': 'he'}),\n",
       "  Munch({'name': 'Alex', 'pron': 'he'}),\n",
       "  Munch({'name': 'Chad', 'pron': 'he'}),\n",
       "  Munch({'name': 'Bradley', 'pron': 'he'}),\n",
       "  Munch({'name': 'Ian', 'pron': 'he'}),\n",
       "  Munch({'name': 'Nathaniel', 'pron': 'he'}),\n",
       "  Munch({'name': 'Liam', 'pron': 'he'}),\n",
       "  Munch({'name': 'Connor', 'pron': 'he'}),\n",
       "  Munch({'name': 'Isaiah', 'pron': 'he'}),\n",
       "  Munch({'name': 'Aiden', 'pron': 'he'}),\n",
       "  Munch({'name': 'Derek', 'pron': 'he'}),\n",
       "  Munch({'name': 'Peter', 'pron': 'he'}),\n",
       "  Munch({'name': 'Dustin', 'pron': 'he'}),\n",
       "  Munch({'name': 'George', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jared', 'pron': 'he'}),\n",
       "  Munch({'name': 'Marcus', 'pron': 'he'}),\n",
       "  Munch({'name': 'Antonio', 'pron': 'he'}),\n",
       "  Munch({'name': 'Henry', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jesus', 'pron': 'he'}),\n",
       "  Munch({'name': 'Jeremiah', 'pron': 'he'}),\n",
       "  Munch({'name': 'Julian', 'pron': 'he'}),\n",
       "  Munch({'name': 'Donald', 'pron': 'he'})]}"
      ]
     },
     "execution_count": 415,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [],
   "source": [
    "import munch\n",
    "editor.lexicons['male_pron'] = [munch.Munch({'name': x, 'pron': 'he'}) for x in editor.lexicons['male']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.template('This is {male}.', return_meta=True, remove_duplicates=True, nsamples=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# editor.template('This is {male_pron1.name}. {male_pron1.pron}.', return_meta=True, remove_duplicates=True, nsamples=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Munch({'name': 'Nathaniel', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Evan', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Andrew', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Christopher', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Richard', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Jeffrey', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Samuel', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Nathan', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Matthew', 'pron': 'he'}),)\n",
      "(Munch({'name': 'Ian', 'pron': 'he'}),)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(['This is Nathaniel. he.',\n",
       "  'This is Evan. he.',\n",
       "  'This is Andrew. he.',\n",
       "  'This is Christopher. he.',\n",
       "  'This is Richard. he.',\n",
       "  'This is Jeffrey. he.',\n",
       "  'This is Samuel. he.',\n",
       "  'This is Nathan. he.',\n",
       "  'This is Matthew. he.',\n",
       "  'This is Ian. he.'],\n",
       " [{'male_pron1': Munch({'name': 'Nathaniel', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Evan', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Andrew', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Christopher', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Richard', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Jeffrey', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Samuel', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Nathan', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Matthew', 'pron': 'he'})},\n",
       "  {'male_pron1': Munch({'name': 'Ian', 'pron': 'he'})}])"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is {male_pron1.name}. {male_pron1.pron}.', return_meta=True, remove_duplicates=True, nsamples=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['This is Kenneth Antonio Peter.',\n",
       "  'This is Richard Austin Hunter.',\n",
       "  'This is Mason Carlos Aiden.',\n",
       "  'This is Bradley Brandon Nathaniel.',\n",
       "  'This is Aiden Jared Derek.',\n",
       "  'This is Luke Paul Adrian.',\n",
       "  'This is Christopher Nathan Nathaniel.',\n",
       "  'This is Gabriel Cody Brian.',\n",
       "  'This is Stephen Henry Logan.',\n",
       "  'This is Paul Daniel Jared.'],\n",
       " [{'male1': 'Kenneth', 'male100': 'Peter', 'male2': 'Antonio'},\n",
       "  {'male1': 'Richard', 'male100': 'Hunter', 'male2': 'Austin'},\n",
       "  {'male1': 'Mason', 'male100': 'Aiden', 'male2': 'Carlos'},\n",
       "  {'male1': 'Bradley', 'male100': 'Nathaniel', 'male2': 'Brandon'},\n",
       "  {'male1': 'Aiden', 'male100': 'Derek', 'male2': 'Jared'},\n",
       "  {'male1': 'Luke', 'male100': 'Adrian', 'male2': 'Paul'},\n",
       "  {'male1': 'Christopher', 'male100': 'Nathaniel', 'male2': 'Nathan'},\n",
       "  {'male1': 'Gabriel', 'male100': 'Brian', 'male2': 'Cody'},\n",
       "  {'male1': 'Stephen', 'male100': 'Logan', 'male2': 'Henry'},\n",
       "  {'male1': 'Paul', 'male100': 'Jared', 'male2': 'Daniel'}])"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is {male1} {male2} {male100}.', return_meta=True, remove_duplicates=True, nsamples=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is a {thing} {thing}'"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping = {'thing' : 'man'}\n",
    "x = 'This is {a:thing} {b:thing}'\n",
    "a = re.compile(r'{([^}]+):([^}]+)}')\n",
    "# 'This is {a:thing}'.format(**mapping)\n",
    "a.search(x).group(1,2)\n",
    "def add_article(noun):\n",
    "    return 'an %s' % noun if noun[0].lower() in ['a', 'e', 'i', 'o', 'u'] else 'a %s' % noun\n",
    "\n",
    "def mysub(match):\n",
    "    options, thing = match.group(1, 2)\n",
    "    ret = '' \n",
    "    if 'a' in options:\n",
    "        ret += '%s ' % add_article(thing).split()[0]\n",
    "    ret += '{%s}' % thing\n",
    "    return ret\n",
    "a.sub(mysub, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('This is a book', 'book'), ('This is an aurora', 'aurora')]"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template(('This is {a:thing}', '{thing}'), thing=['book', 'aurora'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is a very good book.',\n",
       " 'This is a very interesting book.',\n",
       " 'This is a very nice book.',\n",
       " 'This is a very difficult book.',\n",
       " 'This is a very important book.',\n",
       " 'This is a very strange book.',\n",
       " 'This is a very powerful book.',\n",
       " 'This is a very special book.',\n",
       " 'This is a very smart book.',\n",
       " 'This is a very intelligent book.',\n",
       " 'This is a very good movie.',\n",
       " 'This is a very interesting movie.',\n",
       " 'This is a very nice movie.',\n",
       " 'This is a very difficult movie.',\n",
       " 'This is a very important movie.',\n",
       " 'This is a very strange movie.',\n",
       " 'This is a very powerful movie.',\n",
       " 'This is a very special movie.',\n",
       " 'This is a very smart movie.',\n",
       " 'This is a very intelligent movie.',\n",
       " 'This is a very good person.',\n",
       " 'This is a very interesting person.',\n",
       " 'This is a very nice person.',\n",
       " 'This is a very difficult person.',\n",
       " 'This is a very important person.',\n",
       " 'This is a very strange person.',\n",
       " 'This is a very powerful person.',\n",
       " 'This is a very special person.',\n",
       " 'This is a very smart person.',\n",
       " 'This is a very intelligent person.',\n",
       " 'This is a very good student.',\n",
       " 'This is a very interesting student.',\n",
       " 'This is a very nice student.',\n",
       " 'This is a very difficult student.',\n",
       " 'This is a very important student.',\n",
       " 'This is a very strange student.',\n",
       " 'This is a very powerful student.',\n",
       " 'This is a very special student.',\n",
       " 'This is a very smart student.',\n",
       " 'This is a very intelligent student.']"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is a very {bert} {thing}.', thing=['book', 'movie', 'person', 'student'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Jordan has a very good game plan.',\n",
       " 'Patrick has a very good offensive line.',\n",
       " 'Brian has a very good track record.',\n",
       " 'Edward has a very good offensive game.',\n",
       " 'Brian has a very good track record.',\n",
       " 'Zachary has a very good work rate.',\n",
       " 'Jayden has a very good offensive system.',\n",
       " 'Jared has a very good track record.',\n",
       " 'Jesse has a very good offensive system.',\n",
       " 'David has a very good offensive game.',\n",
       " 'Jonathan has a very good offensive game.',\n",
       " 'Logan has a very good skill base.',\n",
       " 'Alexander has a very good track record.',\n",
       " 'Kyle has a very good chess hand.',\n",
       " 'Nathaniel has a very good offensive line.',\n",
       " 'Joshua has a very good skill base.',\n",
       " 'Jared has a very good track record.',\n",
       " 'Alex has a very good offensive game.',\n",
       " 'Nathaniel has a very good offensive game.',\n",
       " 'Aaron has a very good skill base.',\n",
       " 'Jeremy has a very good work rate.',\n",
       " 'Brandon has a very good skill set.',\n",
       " 'Jared has a very good track record.',\n",
       " 'Edward has a very good offensive line.',\n",
       " 'Elijah has a very good offensive line.',\n",
       " 'David has a very good track record.',\n",
       " 'Alexander has a very good track pedigree.',\n",
       " 'Luke has a very good chess hand.',\n",
       " 'Edward has a very good offensive game.',\n",
       " 'Nathaniel has a very good offensive game.',\n",
       " 'Jack has a very good offensive line.',\n",
       " 'Nicholas has a very good track record.',\n",
       " 'Paul has a very good game plan.',\n",
       " 'Peter has a very good chess hand.',\n",
       " 'Alex has a very good chess hand.',\n",
       " 'Jared has a very good track record.',\n",
       " 'Mason has a very good game plan.',\n",
       " 'Nicholas has a very good chess hand.',\n",
       " 'Samuel has a very good chess hand.',\n",
       " 'Alex has a very good offensive line.',\n",
       " 'Brandon has a very good track record.',\n",
       " 'Tyler has a very good offensive system.',\n",
       " 'Samuel has a very good track pedigree.',\n",
       " 'Shawn has a very good track pedigree.',\n",
       " 'Kyle has a very good game plan.',\n",
       " 'Christopher has a very good game plan.',\n",
       " 'Peter has a very good skill base.',\n",
       " 'Jacob has a very good track pedigree.',\n",
       " 'Kevin has a very good work rate.',\n",
       " 'Jeremy has a very good game plan.',\n",
       " 'Mark has a very good offensive system.',\n",
       " 'Aiden has a very good chess hand.',\n",
       " 'Carlos has a very good offensive line.',\n",
       " 'Ethan has a very good offensive game.',\n",
       " 'Ethan has a very good offensive system.',\n",
       " 'Mark has a very good chess hand.',\n",
       " 'Sean has a very good track record.',\n",
       " 'Adam has a very good track pedigree.',\n",
       " 'Isaiah has a very good work rate.',\n",
       " 'Edward has a very good work rate.',\n",
       " 'Joseph has a very good track pedigree.',\n",
       " 'Ryan has a very good track pedigree.',\n",
       " 'Jack has a very good skill base.',\n",
       " 'Michael has a very good track record.',\n",
       " 'Robert has a very good track record.',\n",
       " 'Nathan has a very good offensive system.',\n",
       " 'Kenneth has a very good skill set.',\n",
       " 'Hunter has a very good track pedigree.',\n",
       " 'Stephen has a very good skill set.',\n",
       " 'Nathan has a very good skill base.',\n",
       " 'Gabriel has a very good offensive line.',\n",
       " 'Jack has a very good game plan.',\n",
       " 'Joseph has a very good skill set.',\n",
       " 'Noah has a very good work rate.',\n",
       " 'Jordan has a very good work rate.',\n",
       " 'Jesse has a very good track pedigree.',\n",
       " 'Gabriel has a very good work rate.',\n",
       " 'Ian has a very good game plan.',\n",
       " 'Charles has a very good track pedigree.',\n",
       " 'Julian has a very good skill base.',\n",
       " 'Christian has a very good work rate.',\n",
       " 'Isaiah has a very good work rate.',\n",
       " 'Scott has a very good skill base.',\n",
       " 'Jeremy has a very good offensive line.',\n",
       " 'Cody has a very good track record.',\n",
       " 'Jonathan has a very good game plan.',\n",
       " 'Hunter has a very good game plan.',\n",
       " 'Jesse has a very good offensive game.',\n",
       " 'John has a very good skill set.',\n",
       " 'Kevin has a very good track record.',\n",
       " 'Patrick has a very good offensive system.',\n",
       " 'Julian has a very good skill base.',\n",
       " 'Adam has a very good chess hand.',\n",
       " 'Christian has a very good game plan.',\n",
       " 'Shawn has a very good offensive system.',\n",
       " 'Christopher has a very good offensive game.',\n",
       " 'Jackson has a very good offensive line.',\n",
       " 'Luis has a very good game plan.',\n",
       " 'Peter has a very good skill base.',\n",
       " 'John has a very good skill set.']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('{male} has a very good {bert} {bert}.', thing=['movie', 'book', 'tv show', 'person', 'dog', 'country'], nsamples=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(['good'], ' This is a good movie.', 15.688888549804688),\n",
       " (['great'], ' This is a great movie.', 14.89714241027832),\n",
       " (['bad'], ' This is a bad movie.', 14.015711784362793),\n",
       " (['terrible'], ' This is a terrible movie.', 13.26148796081543),\n",
       " (['nice'], ' This is a nice movie.', 12.802513122558594),\n",
       " (['horror'], ' This is a horror movie.', 12.745840072631836),\n",
       " (['scary'], ' This is a scary movie.', 12.734757423400879),\n",
       " (['sad'], ' This is a sad movie.', 12.687625885009766),\n",
       " (['beautiful'], ' This is a beautiful movie.', 12.429882049560547),\n",
       " (['funny'], ' This is a funny movie.', 12.342477798461914),\n",
       " (['wonderful'], ' This is a wonderful movie.', 12.230314254760742),\n",
       " (['short'], ' This is a short movie.', 12.098125457763672),\n",
       " (['cult'], ' This is a cult movie.', 12.094284057617188),\n",
       " (['fun'], ' This is a fun movie.', 11.956672668457031),\n",
       " (['horrible'], ' This is a horrible movie.', 11.884394645690918),\n",
       " (['fantastic'], ' This is a fantastic movie.', 11.882230758666992),\n",
       " (['weird'], ' This is a weird movie.', 11.824714660644531),\n",
       " (['violent'], ' This is a violent movie.', 11.688575744628906),\n",
       " (['stupid'], ' This is a stupid movie.', 11.66788387298584),\n",
       " (['cool'], ' This is a cool movie.', 11.60772705078125)]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.tg.unmask('This is a <mask> movie.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is not a very good movie',\n",
       " 'This is not a very nice movie',\n",
       " 'This is not a very funny movie',\n",
       " 'This is not a very bad movie',\n",
       " 'This is not a very great movie',\n",
       " 'This is not a very violent movie',\n",
       " 'This is not a very fun movie',\n",
       " 'This is not a very long movie',\n",
       " 'This is not a very interesting movie',\n",
       " 'This is not a very smart movie']"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is not a very {bert} {b}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'str1': 'This is not the best restaurant in the world',\n",
       "  'str2': 'I think this is the best place in the world',\n",
       "  'str3': 'This is a horrible movie.',\n",
       "  'str4': \"This movie is not good, it's horrible\"},\n",
       " {'str1': 'This is not the best shot in the campus',\n",
       "  'str2': 'I think this is the best place in the campus',\n",
       "  'str3': 'This is a stupid movie.',\n",
       "  'str4': \"This movie is not good, it's stupid\"},\n",
       " {'str1': 'This is not the best place in the library',\n",
       "  'str2': 'I think this is the best place in the library',\n",
       "  'str3': 'This is a bad movie.',\n",
       "  'str4': \"This movie is not good, it's bad\"},\n",
       " {'str1': 'This is not the worst thing in the world',\n",
       "  'str2': 'I think this is the worst place in the world',\n",
       "  'str3': 'This is a awful movie.',\n",
       "  'str4': \"This movie is not good, it's awful\"},\n",
       " {'str1': 'This is not the best view in the place',\n",
       "  'str2': 'I think this is the best place in the place',\n",
       "  'str3': 'This is a crap movie.',\n",
       "  'str4': \"This movie is not good, it's crap\"}]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = editor.template({'str1': 'This is not the {bert} {bert} in the {a}',\n",
    "                 'str2': 'I think this is the {bert} place in the {a}', \n",
    "                 'str3': 'This is a {bert1} movie.',\n",
    "                 'str4': 'This movie is not good, it\\'s {bert1}'},\n",
    "                a=['world', 'library', 'campus', 'place'], return_meta=False, nsamples=500)\n",
    "a[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is not the safest place in the library',\n",
       " 'This is not the worst place in the place',\n",
       " 'This is not the only one in the library',\n",
       " 'This is not the safest room in the world',\n",
       " 'This is not the best place in the campus',\n",
       " 'This is not the worst place in the world',\n",
       " 'This is not the worst thing in the world',\n",
       " 'This is not the safest room in the world',\n",
       " 'This is not the worst thing in the campus',\n",
       " 'This is not the safest spot in the world',\n",
       " 'This is not the safest spot in the campus',\n",
       " 'This is not the safest place in the campus',\n",
       " 'This is not the biggest thing in the world',\n",
       " 'This is not the best place in the library',\n",
       " 'This is not the worst thing in the campus',\n",
       " 'This is not the biggest problem in the place',\n",
       " 'This is not the biggest problem in the place',\n",
       " 'This is not the only one in the world',\n",
       " 'This is not the only problem in the campus',\n",
       " 'This is not the best place in the campus']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is not the {bert} {bert} in the {a}',  a=['world', 'library', 'campus', 'place'], return_meta=False, nsamples=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 877,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.template('This is not BERT bad {a}.',a=['a', 'b'], return_meta=False, nsamples=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['This is VERYLONGTOKENTHATWILLNOTEXISTEVER not VERYLONGTOKENTHATWILLNOTEXISTEVER VERYLONGTOKENTHATWILLNOTEXISTEVER bad.']\n",
      "['This is VERYLONGTOKENTHATWILLNOTEXISTEVER not VERYLONGTOKENTHATWILLNOTEXISTEVER VERYLONGTOKENTHATWILLNOTEXISTEVER bad.']\n",
      "['This is <mask> not <mask> <mask> bad.']\n",
      "aefjdksalf\n",
      "[['really', 'all', 'that'], ['actually', 'all', 'that'], ['probably', 'all', 'that'], ['honestly', 'all', 'that'], ['still', 'all', 'that'], ['really', 'half', '-'], ['actually', 'half', '-'], ['actually', 'nearly', 'as'], ['actually', 'half', 'as'], ['actually', 'quite', 'so']]\n",
      "This is {bert[0]} not {bert[1]} {bert[2]} bad.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(['This is really not all that bad.',\n",
       "  'This is actually not all that bad.',\n",
       "  'This is probably not all that bad.',\n",
       "  'This is honestly not all that bad.',\n",
       "  'This is still not all that bad.',\n",
       "  'This is really not half - bad.',\n",
       "  'This is actually not half - bad.',\n",
       "  'This is actually not nearly as bad.',\n",
       "  'This is actually not half as bad.',\n",
       "  'This is actually not quite so bad.'],\n",
       " [{'bert': ['really', 'all', 'that']},\n",
       "  {'bert': ['actually', 'all', 'that']},\n",
       "  {'bert': ['probably', 'all', 'that']},\n",
       "  {'bert': ['honestly', 'all', 'that']},\n",
       "  {'bert': ['still', 'all', 'that']},\n",
       "  {'bert': ['really', 'half', '-']},\n",
       "  {'bert': ['actually', 'half', '-']},\n",
       "  {'bert': ['actually', 'nearly', 'as']},\n",
       "  {'bert': ['actually', 'half', 'as']},\n",
       "  {'bert': ['actually', 'quite', 'so']}])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is {bert} not {bert} {bert} bad.', return_meta=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "nouns = ['flight', 'seat', 'pilot', 'staff', 'plane', 'airline', 'cabin crew', 'aircraft', 'food']\n",
    "pos = ['liked', 'enjoyed', 'loved', 'admired', 'appreciated']\n",
    "neg = ['hated', 'disliked', 'regretted']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tg' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-c38380a635c0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meditor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'I really <mask> the {n}.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnouns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munmask_multiple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeam_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# tg.unmask_multiple(ts, metric='avg', beam_size=100)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tg' is not defined"
     ]
    }
   ],
   "source": [
    "ts = editor.template('I really <mask> the {n}.', n=nouns)\n",
    "print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000)][:20])\n",
    "# tg.unmask_multiple(ts, metric='avg', beam_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'We', 'He', 'he', 'She', 'They', 'we', 'and', '...', 'she', 'they', 'Everyone', 'i', 'People', 'You', 'Alex', 'Everybody', 'And', 'Dad', 'John']\n"
     ]
    }
   ],
   "source": [
    "ts = editor.template('<mask> really {pn} the {n}.', n=nouns, pn=pos+neg)\n",
    "print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000)][:20])\n",
    "# tg.unmask_multiple(ts, metric='avg', beam_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "subj = ['I', 'We', 'They', 'we', 'He', 'he', 'She', 'she', 'they', 'people', 'People', 'you', '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tg' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-93-31178ff2f6d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meditor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'I didn\\'t like the {n}, it was very <mask>.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnouns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munmask_multiple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeam_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'avg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# tg.unmask_multiple(ts, metric='avg', beam_size=100)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tg' is not defined"
     ]
    }
   ],
   "source": [
    "ts = editor.template('I didn\\'t like the {n}, it was very <mask>.', n=nouns)\n",
    "print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000, metric='avg')][:100])\n",
    "# tg.unmask_multiple(ts, metric='avg', beam_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_adj = ['good', 'nice', 'helpful', 'comfortable', 'cool', 'efficient', 'pleasant', 'interesting', 'impressive', 'welcoming', 'professional', 'beautiful', 'exciting', 'positive', 'solid', 'amazing', 'wonderful', 'lovely']\n",
    "neg_adj = ['bad', 'boring', 'unpleasant', 'difficult', 'uncomfortable', 'ugly', 'poor', 'disappointing', 'sad', 'annoying', 'dirty', 'frustrating', 'depressing', 'nasty', 'horrible', 'stupid', 'negative', 'awful', 'stressful', 'irritating', 'disgusting']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_eg = editor.template('{sub}{really} {p} the {n}.', sub=subj, really=[' really', ''], n=nouns, p=pos)\n",
    "pos_eg += editor.template('The {n} was{very} {p}.', sub=subj, very=[' very', ''], n=nouns, p=pos_adj)\n",
    "neg_eg = editor.template('{sub}{really} {p} the {n}.', sub=subj, really=[' really', ''], n=nouns, p=neg)\n",
    "neg_eg += editor.template('The {n} was{very} {p}.', sub=subj, very=[' very', ''], n=nouns, p=neg_adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1494, 1080)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pos_eg), len(neg_eg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pos_eg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pos = Mft(pos_eg, labels=1)\n",
    "test_neg = Mft(neg_eg, labels=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_pos.set_expect(Expect.single(Expect.eq()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1494 examples\n",
      "Predicting 1080 examples\n"
     ]
    }
   ],
   "source": [
    "test_pos.run(pp)\n",
    "test_neg.run(pp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     1080\n",
      "Failure rate:   1.7%\n",
      "\n",
      "Example fails:\n",
      "1 (1.0) The cabin crew was very stressful.\n",
      "1 (1.0) The flight was very stressful.\n",
      "1 (1.0) The flight was stressful.\n",
      "1 (1.0) The aircraft was stressful.\n",
      "1 (1.0) The seat was very stressful.\n"
     ]
    }
   ],
   "source": [
    "test_neg.summary(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     1494\n",
      "Failure rate:   0.0%\n"
     ]
    }
   ],
   "source": [
    "test_pos.summary(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fairness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### As invariance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 701,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj = [x[0][0] for x in tg.unmask('John is a black <mask>.')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 720,
   "metadata": {},
   "outputs": [],
   "source": [
    "races = ['black', 'white', 'hispanic', 'asian']\n",
    "d = editor.template(['{male} is a %s {adj}.' % r for r in races], adj=adj, return_maps=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 713,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 8000 examples\n"
     ]
    }
   ],
   "source": [
    "test = Inv(d, threshold=0.1)\n",
    "test.run(pp, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 714,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     2000\n",
      "Failure rate:   95.0%\n",
      "\n",
      "Example fails:\n",
      "0 (0.1) Gabriel is a black Muslim.\n",
      "1 (0.9) Gabriel is a hispanic Muslim.\n",
      "1 (0.9) Gabriel is a asian Muslim.\n",
      "\n",
      "0 (0.0) James is a black person.\n",
      "1 (0.9) James is a hispanic person.\n",
      "1 (0.9) James is a asian person.\n",
      "\n",
      "0 (0.1) Jason is a black girl.\n",
      "1 (1.0) Jason is a hispanic girl.\n",
      "1 (1.0) Jason is a asian girl.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 727,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12 black\n",
      "0.19 white\n",
      "0.96 hispanic\n",
      "0.98 asian\n"
     ]
    }
   ],
   "source": [
    "pps = np.array([[x[i][1] for i in range(4)] for x in test.results.confs])\n",
    "for i in range(4):\n",
    "    print('%.2f %s' % (pps[:, i].mean(), races[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### as MFT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "metadata": {},
   "outputs": [],
   "source": [
    "d, m = editor.template('{male} is a {r} {adj}.', r=races,  adj=adj, return_maps=True, nsamples=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 731,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'r': 'black', 'adj': 'man', 'male': 'Michael'}"
      ]
     },
     "execution_count": 731,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 773,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([None], dtype=object)"
      ]
     },
     "execution_count": 773,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 777,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "def is_fair(data, preds, confs, labels, metas):\n",
    "    rets = collections.defaultdict(lambda: [])\n",
    "    ret = np.repeat(None, len(data))\n",
    "    for c, m in zip(confs, metas):\n",
    "        rets[m['r']].append(c[1])\n",
    "    for i, r in enumerate(rets):\n",
    "        rets[r] = np.mean(rets[r])\n",
    "        ret[i] = rets[r]\n",
    "    print(rets)\n",
    "    return ret \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 778,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 100 examples\n",
      "defaultdict(<function is_fair.<locals>.<lambda> at 0x7f126044f598>, {'white': 0.356781789360361, 'asian': 0.9733978605270386, 'hispanic': 0.9593281026544243, 'black': 0.12038368929643184})\n"
     ]
    }
   ],
   "source": [
    "ex = Expect.test(is_fair)\n",
    "test = Mft(d, expect=ex, meta=m)\n",
    "test.run(pp, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a bad idea, fairness tests should just be INVs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inv, dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "r = csv.DictReader(open('/home/marcotcr/datasets/airline/Tweets.csv'))\n",
    "labels = []\n",
    "confs = []\n",
    "airlines = []\n",
    "tdata = []\n",
    "reasons = []\n",
    "for row in r:\n",
    "    sentiment, conf, airline, text = row['airline_sentiment'], row['airline_sentiment_confidence'], row['airline'], row['text']\n",
    "    labels.append(sentiment)\n",
    "    confs.append(conf)\n",
    "    airlines.append(airline)\n",
    "    tdata.append(text)\n",
    "    reasons.append(row['negativereason'])\n",
    "\n",
    "mapping = {'negative': 0, 'positive': 2, 'neutral': 1}\n",
    "labels = np.array([mapping[x] for x in labels]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = tdata\n",
    "parsed_data = list(nlp.pipe(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_typo(string):\n",
    "    string = list(string)\n",
    "    swaps = 1\n",
    "    swaps = np.random.choice(len(string) - 1, swaps)\n",
    "    for swap in swaps:\n",
    "        swap = np.random.choice(len(string) - 1)\n",
    "        tmp = string[swap]\n",
    "        string[swap] = string[swap + 1]\n",
    "        string[swap + 1] = tmp\n",
    "    return ''.join(string)\n",
    "\n",
    "def add_typos(string):\n",
    "    return list(set([add_typo(string) for _ in range(10)]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [],
   "source": [
    "from checklist.perturb import Perturb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = Perturb.perturb(np.random.choice(sentences, 100), add_typos, keep_original=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1043 examples\n"
     ]
    }
   ],
   "source": [
    "test = Inv(data, threshold=0.1)\n",
    "test.run(pp, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     100\n",
      "Failure rate:   24.0%\n",
      "\n",
      "Example fails:\n",
      "0.0 @JetBlue Not for the dates or destination I'm headed 😔\n",
      "0.5 @JetBlu eNot for the dates or destination I'm headed 😔\n",
      "\n",
      "0.5 @AmericanAir will do. I also passed the website around to other passengers.\n",
      "0.7 @AmericanAir will do. I also passed the website around to other psasengers.\n",
      "0.7 @AmericanAir will do. I also passed the ewbsite around to other passengers.\n",
      "\n",
      "1.0 @united DM'ed you\n",
      "0.1 @uinted DM'ed you\n",
      "0.1 @united DM'ed yuo\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 2000 examples\n"
     ]
    }
   ],
   "source": [
    "data = Perturb.perturb(np.random.choice(sentences, 1000), add_typo, keep_original=True)\n",
    "test = Inv(data, threshold=0.1)\n",
    "test.run(pp, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     1000\n",
      "Failure rate:   4.9%\n",
      "\n",
      "Example fails:\n",
      "0.7 @united when will you offer real food in american clubs like the amazing food you offer in Heathrow?\n",
      "0.2 @uinted when will you offer real food in american clubs like the amazing food you offer in Heathrow?\n",
      "\n",
      "1.0 @united maybe on my return trip 👍\n",
      "0.1 @united maybe on my erturn trip 👍\n",
      "\n",
      "0.7 @united was in the air. Just DMd you\n",
      "0.0 @united was in the ai.r Just DMd you\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: get from somethwere else\n",
    "def add_negatives(string):\n",
    "    string = string.strip('.')\n",
    "    return [string + '. ' + l for l in ['I hate you', 'I despise you', 'You suck']]\n",
    "def add_positive(string):\n",
    "    string = string.strip('.')\n",
    "    return [string + '. ' + l for l in ['I love you', 'I like you', 'You are great']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 791,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 4000 examples\n",
      "Test cases:     1000\n",
      "Filtered cases: 259 (25.9%)\n",
      "Failure rate:   0.0%\n"
     ]
    }
   ],
   "source": [
    "data = Perturb.perturb(np.random.choice(sentences, 1000), add_negatives, keep_original=True)\n",
    "expect_fn = Expect.monotonic(1, increasing=False, tolerance=0.1)\n",
    "test = Dir(data, expect_fn)\n",
    "test.run(pp, overwrite=True)\n",
    "test.set_monotonic_print(label=1, increasing=False)\n",
    "test.summary(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 808,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 400 examples\n",
      "Test cases:     100\n",
      "Filtered cases: 72 (72.0%)\n",
      "Failure rate:   0.0%\n"
     ]
    }
   ],
   "source": [
    "data = Perturb.perturb(np.random.choice(sentences, 100), add_positive, keep_original=True)\n",
    "expect_fn = Expect.monotonic(1, increasing=True, tolerance=0.1)\n",
    "test = Dir(data, expect_fn)\n",
    "test.run(pp, overwrite=True)\n",
    "test.set_monotonic_print(label=1, increasing=True)\n",
    "test.summary(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### QQP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {},
   "outputs": [],
   "source": [
    "qqp = model_wrapper.ModelWrapper()\n",
    "pp = PredictorWrapper.wrap_softmax(qqp.predict_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"\tWhat does the Quran say about homosexuality?\t0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "qs = []\n",
    "labels = []\n",
    "all_questions = set()\n",
    "for x in open('/home/marcotcr/datasets/glue/glue_data/QQP/dev.tsv').readlines()[1:]:\n",
    "    try:\n",
    "        q1, q2, label = x.strip().split('\\t')[3:]\n",
    "    except:\n",
    "        print(x)\n",
    "        continue\n",
    "    all_questions.add(q1)\n",
    "    all_questions.add(q2)\n",
    "    qs.append((q1, q2))\n",
    "    labels.append(label)\n",
    "labels = np.array(labels).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "quran = [x[0][0] for x in tg.unmask('What does the Quran say about <mask>?')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "quran.remove('homosexuals')\n",
    "quran.remove('gays')\n",
    "quran.remove('this')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 515,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 272 examples\n",
      "Test cases:     272\n",
      "Failure rate:   2.2%\n",
      "\n",
      "Example fails:\n",
      "0.6 ('What does the Quran say about polygamy?', 'What does the Quran say about circumcision?')\n",
      "0.7 ('What does the Quran say about circumcision?', 'What does the Quran say about polygamy?')\n",
      "0.8 ('What does the Quran say about Muhammad?', 'What does the Quran say about Muslims?')\n",
      "1.0 ('What does the Quran say about Islam?', 'What does the Quran say about Muslims?')\n",
      "0.8 ('What does the Quran say about Muslims?', 'What does the Quran say about Islam?')\n"
     ]
    }
   ],
   "source": [
    "data = editor.template(('What does the Quran say about {thing1}?', 'What does the Quran say about {thing2}?'),\n",
    "                thing1=quran, thing2=quran, remove_duplicates=True)\n",
    "test = Mft(data, labels=0)\n",
    "test.run(pp, overwrite=True)\n",
    "test.summary(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Bible', 'book', 'report', 'survey', 'poll', 'study', 'Constitution', 'data', 'bible', 'election', 'census', 'law', 'UN', 'Quran', 'evidence', 'science', 'constitution', 'Koran', 'vote', 'bill', 'film', 'Holocaust', 'research', 'Pope', 'letter', 'article', 'verdict', 'world', 'movie', 'government', 'video', 'Torah', 'song', 'moon', 'public', 'dictionary', 'president', 'referendum', 'ACA', 'eclipse', 'church', 'US', 'media', 'Church', 'pope', 'record', 'test', 'answer', 'literature', 'Prophet', 'CDC', 'FDA', 'WHO', 'scripture', 'community', 'Buddha', 'ban', 'prophet', 'statement', 'interview', 'Gospel', 'Book', 'past', 'question', 'decision']\n"
     ]
    }
   ],
   "source": [
    "print([x[0][0] for x in tg.unmask_multiple(editor.template('What does the <mask> say about {thing}?', thing=quran))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 516,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1530 examples\n",
      "Test cases:     1530\n",
      "Failure rate:   8.9%\n",
      "\n",
      "Example fails:\n",
      "0.6 ('What does the Bible say about homosexuality?', 'What does the Church say about homosexuality?')\n",
      "1.0 ('What does the Gospel say about you?', 'What does the Bible say about you?')\n",
      "0.9 ('What does the Quran say about religion?', 'What does the Prophet say about religion?')\n",
      "0.6 ('What does the Bible say about women?', 'What does the Torah say about women?')\n",
      "1.0 ('What does the Bible say about God?', 'What does the Gospel say about God?')\n"
     ]
    }
   ],
   "source": [
    "books = ['Bible', 'Constitution', 'Quran', 'Pope', 'Torah', 'Church', 'Buddha', 'Prophet', 'Gospel', 'Book']\n",
    "data = editor.template(('What does the {book1} say about {thing}?', 'What does the {book2} say about {thing}?'),\n",
    "                thing=quran, book1=books, book2=books, remove_duplicates=True)\n",
    "test = Mft(data, labels=0)\n",
    "test.run(pp, overwrite=True)\n",
    "test.summary(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inv, dirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_map =  pickle.load(open('/home/marcotcr/tmp/processed_qqp.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_qs = [spacy_map[x[0]] for x in qs] + [spacy_map[x[1]] for x in qs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_pairs = [(spacy_map[x[0]],spacy_map[x[1]]) for x in qs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_names(pair):\n",
    "    x0, x1 = pair\n",
    "    ents_0 = set([x[0].text for x in x0.ents if x[0].ent_type_ == 'PERSON'])\n",
    "    ents_1 = set([x[0].text for x in x1.ents if x[0].ent_type_ == 'PERSON'])\n",
    "    ret = []\n",
    "    if ents_0 and ents_0.intersection(ents_1):\n",
    "        for e in ents_0.intersection(ents_1):\n",
    "            if e == 'Quora':\n",
    "                continue\n",
    "            ret.extend([(pair[0].text.replace(e, n), pair[1].text.replace(e, n)) for n in editor.lexicons['male'] + editor.lexicons['female']])\n",
    "    if ret:\n",
    "        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)\n",
    "        return [ret[i] for i in idxs]\n",
    "# change_names(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 530,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 9792 examples\n",
      "Test cases:     1632\n",
      "Failure rate:   11.1%\n",
      "\n",
      "Example fails:\n",
      "0.2 (\"What is India's relationship with Bangladesh?\", \"What is Bangladesh's relationship with India?\")\n",
      "0.9 (\"What is India's relationship with Isabella?\", \"What is Isabella's relationship with India?\")\n",
      "0.9 (\"What is India's relationship with Jesus?\", \"What is Jesus's relationship with India?\")\n",
      "\n",
      "1.0 ('Which is best place to stay and visit in Kerala?', 'What are the best 10 places to visit in Kerala including any falls?')\n",
      "0.4 ('Which is best place to stay and visit in Sara?', 'What are the best 10 places to visit in Sara including any falls?')\n",
      "0.4 ('Which is best place to stay and visit in Emily?', 'What are the best 10 places to visit in Emily including any falls?')\n",
      "\n",
      "1.0 ('What is Jake Williams’s history that made him into a narcissist?', 'How is Jake Williams a narcissist?')\n",
      "0.3 ('What is Sara Williams’s history that made him into a narcissist?', 'How is Sara Williams a narcissist?')\n",
      "0.4 ('What is Nicole Williams’s history that made him into a narcissist?', 'How is Nicole Williams a narcissist?')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "idxs = np.random.choice(len(processed_pairs), 2000,  replace=False)\n",
    "# idxs = np.random.choice(len(processed_pairs), len(processed_pairs),  replace=False)\n",
    "sl = [processed_pairs[i] for i in idxs]\n",
    "data = Perturb.perturb(sl, change_names, keep_original=True)\n",
    "# test = Dir(data, Expect.wrap(Expect.all(Expect.pairwise_to_group(Expect.monotonic_label(1, increasing=True, tolerance=0)))) )\n",
    "test = Inv(data, threshold=0.1)\n",
    "test.run(pp, overwrite=True)\n",
    "test.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 432 examples\n",
      "Test cases:     72\n",
      "Failure rate:   5.6%\n",
      "\n",
      "Example fails:\n",
      "1.0 ('What are the best places to visit in Wayanad, Kerala?', 'What can be the medium budget to visit best places in Kerala for three members (2-3 days)?')\n",
      "0.2 ('What are the best places to visit in Wayanad, Isaiah?', 'What can be the medium budget to visit best places in Isaiah for three members (2-3 days)?')\n",
      "0.0 ('What are the best places to visit in Wayanad, Jamie?', 'What can be the medium budget to visit best places in Jamie for three members (2-3 days)?')\n",
      "\n",
      "0.9 ('Was Donald Trump always rich?', 'How rich is Donald Trump?')\n",
      "0.5 ('Was Jason Trump always rich?', 'How rich is Jason Trump?')\n",
      "\n",
      "1.0 ('What is the best way to start contributing to the Linux kernel?', 'How should I start contributing for Linux?')\n",
      "0.2 ('What is the best way to start contributing to the Gabriel kernel?', 'How should I start contributing for Gabriel?')\n",
      "0.0 ('What is the best way to start contributing to the Elizabeth kernel?', 'How should I start contributing for Elizabeth?')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "idxs = np.random.choice(len(processed_pairs), 2000,  replace=False)\n",
    "# idxs = np.random.choice(len(processed_pairs), len(processed_pairs),  replace=False)\n",
    "sl = [processed_pairs[i] for i in idxs]\n",
    "data = Perturb.perturb(sl, change_names, keep_original=True)\n",
    "# test = Dir(data, Expect.wrap(Expect.all(Expect.pairwise_to_group(Expect.monotonic_label(1, increasing=True, tolerance=0)))) )\n",
    "test = Inv(data, threshold=0.1)\n",
    "test.run(pp, overwrite=True)\n",
    "test.summary(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test.results.passed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_name_in_one(pair):\n",
    "    x0, x1 = pair\n",
    "    ents_0 = set([x[0].text for x in x0.ents if x[0].ent_type_ == 'PERSON'])\n",
    "    ents_1 = set([x[0].text for x in x1.ents if x[0].ent_type_ == 'PERSON'])\n",
    "    ret = []\n",
    "    if ents_0 and ents_0.intersection(ents_1):\n",
    "        for e in ents_0.intersection(ents_1):\n",
    "            if e == 'Quora':\n",
    "                continue\n",
    "            ret.extend([(pair[0].text.replace(e, n), pair[1].text) for n in editor.lexicons['male'] + editor.lexicons['female']])\n",
    "            ret.extend([(pair[0].text, pair[1].text.replace(e, n)) for n in editor.lexicons['male'] + editor.lexicons['female']])\n",
    "    if ret:\n",
    "        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)\n",
    "        return [ret[i] for i in idxs]\n",
    "# change_names(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 756 examples\n",
      "Test cases:     126\n",
      "Filtered cases: 64 (50.8%)\n",
      "Failure rate:   3.2%\n",
      "\n",
      "Example fails:\n",
      "0.1 ('Why does Donald Trump think the debate schedule favors Hillary?', 'Why is Donald Trump saying the debate schedule is unfair? Is it because his support base would rather watch the NFL than his debate?')\n",
      "0.2 ('Why does Donald Trump think the debate schedule favors Hillary?', 'Why is Ava Trump saying the debate schedule is unfair? Is it because his support base would rather watch the NFL than his debate?')\n",
      "\n",
      "0.5 ('What do you feel about Donald Trump winning the elections?', 'How do you feel about Donald Trump winning the Republican nomination?')\n",
      "1.0 ('What do you feel about Donald Trump winning the elections?', 'How do you feel about Adam Trump winning the Republican nomination?')\n",
      "1.0 ('What do you feel about Donald Trump winning the elections?', 'How do you feel about Austin Trump winning the Republican nomination?')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "monotonic = Expect.monotonic(1, increasing=False, tolerance=0.1)\n",
    "idxs = np.random.choice(len(processed_pairs), 3000, replace=False)\n",
    "sl = [processed_pairs[i] for i in idxs]\n",
    "data = Perturb.perturb(sl, change_name_in_one, keep_original=True)\n",
    "test = Dir(data, monotonic)\n",
    "test.run(pp, overwrite=True)\n",
    "test.set_monotonic_print(label=1, increasing=False)\n",
    "test.summary(n=3)\n",
    "# test.results.passed[test.results.passed != None].astype(bool).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 625,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True, False, False, False])"
      ]
     },
     "execution_count": 625,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([True, False, -1, 0, ]) >= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if only consider examples that were duplicates\n",
    "sl_fn = lambda pred, *args, **kwargs: pred == 1\n",
    "# This substitutes the previous one\n",
    "# def sl_fn(x, pred, *args, **kwargs):\n",
    "#     print(pred)\n",
    "#     return pred[0] == 1\n",
    "sl_fn2 = lambda x, pred, *args, **kwargs: pred[0] == 1\n",
    "is_false = Expect.single(Expect.eq(0), agg_fn=Expect.all(ignore_first=True))#, slice_fn=sl_fn)\n",
    "# is_false = Expect.slice_pairwise(is_false, sl_fn)\n",
    "is_false = Expect.slice_testcase(is_false, sl_fn2)\n",
    "test.set_expect(is_false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     126\n",
      "Filtered cases: 53 (42.1%)\n",
      "Failure rate:   52.1%\n",
      "\n",
      "Example fails:\n",
      "1.0 ('What are the best books on Joseph Goebbels?', 'Are there any really, really interesting books on Joseph Goebbels?')\n",
      "1.0 ('What are the best books on Peter Goebbels?', 'Are there any really, really interesting books on Joseph Goebbels?')\n",
      "1.0 ('What are the best books on Jesse Goebbels?', 'Are there any really, really interesting books on Joseph Goebbels?')\n",
      "\n",
      "1.0 ('How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sahara?', 'How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sonoran Desert?')\n",
      "1.0 ('How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sahara?', 'How cold can Tiffany Gobi Desert get, and how do its average temperatures compare to Tiffany ones in Tiffany Sonoran Desert?')\n",
      "1.0 ('How cold can Ella Gobi Desert get, and how do its average temperatures compare to Ella ones in Ella Sahara?', 'How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sonoran Desert?')\n",
      "\n",
      "0.9 ('Should I read A Song of Ice and Fire or watch Game of Thrones first?', 'Should I read A Song of Ice and Fire after watching the Game of Thrones TV series?')\n",
      "0.9 ('Should I read Ian Song of Ice and Fire or watch Game of Thrones first?', 'Should I read A Song of Ice and Fire after watching the Game of Thrones TV series?')\n",
      "0.9 ('Should I read Christina Song of Ice and Fire or watch Game of Thrones first?', 'Should I read A Song of Ice and Fire after watching the Game of Thrones TV series?')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from checklist.inv_dir import pairwise_print_fn\n",
    "def fail_cr(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    return pred != 0\n",
    "def sort_cr(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    return orig_conf[0]\n",
    "print_fn = pairwise_print_fn(fail_cr, sort_cr)\n",
    "test.summary(n=3, print_fn=print_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Squad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mltests import bert_squad_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = bert_squad_model.BertSquad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Just makes confidence=1 for every prediction\n",
    "pp = PredictorWrapper.wrap_predict(model.predict_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c2415e533d24776ac9f9d70c9e2c10e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Mary']"
      ]
     },
     "execution_count": 580,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_pairs([('Who is dumb?', 'Mary is somewhat dumb. John is not so dumb.')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00f75826c94941a5a642cf712e25dfcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(['Mary'], array([1.]))"
      ]
     },
     "execution_count": 581,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp([('Who is dumb?', 'Mary is somewhat dumb. John is not so dumb.')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.lexicons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['drinking', 'smoking', 'crying', 'reading', 'writing', 'worrying', 'eating', 'driving', 'listening', 'praying', 'talking', 'working', 'trying', 'calling', 'laughing']\n"
     ]
    }
   ],
   "source": [
    "verbs = [x[0][0] for x in tg.unmask('Luke told Mary that she should probably stop <mask>.')]\n",
    "verbs = [x for x in verbs if 'ing' in x]\n",
    "print(verbs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 584,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = []\n",
    "data, maps = editor.template(('Who is currently {verb}?', '{male} told {female} she should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)\n",
    "labels += [m['female'] for m in maps]\n",
    "n, maps = editor.template(('Who is currently {verb}?', '{male} told {female} he should probably stop {verb}'), nsamples=100,verb=verbs, return_maps=True)\n",
    "labels += [m['male'] for m in maps]\n",
    "data += n\n",
    "n, maps = editor.template(('Who is currently {verb}?', '{female} told {male} he should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)\n",
    "labels += [m['male'] for m in maps]\n",
    "data += n\n",
    "n, maps = editor.template(('Who is currently {verb}?', '{female} told {male} she should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)\n",
    "labels += [m['female'] for m in maps]\n",
    "data += n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 585,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Who is currently worrying?',\n",
       " 'Juan told Crystal she should probably stop worrying')"
      ]
     },
     "execution_count": 585,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 400 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22f8fa8a43d1438cbc766449fd7abdab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test = Mft(data, labels=labels)\n",
    "test.run(pp)\n",
    "# test.results.passed.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 609,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_squad_with_context(x, pred, conf, label=None, *args, **kwargs):\n",
    "    q, c = x\n",
    "    ret = 'C: %s\\nQ: %s\\n' % (c, q)\n",
    "    if label is not None:\n",
    "        ret += 'A: %s\\n' % label\n",
    "    ret += 'P: %s\\n' % pred\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 613,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_squad(x, pred, conf, label=None, *args, **kwargs):\n",
    "    q, c = x\n",
    "    ret = 'Q: %s\\n' % (q)\n",
    "    if label is not None:\n",
    "        ret += 'A: %s\\n' % label\n",
    "    ret += 'P: %s\\n' % pred\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 615,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:     400\n",
      "Failure rate:   58.2%\n",
      "\n",
      "Example fails:\n",
      "C: Taylor told Carlos she should probably stop praying\n",
      "Q: Who is currently praying?\n",
      "A: Taylor\n",
      "P: Carlos\n",
      "\n",
      "C: Carlos told Rachel he should probably stop worrying\n",
      "Q: Who is currently worrying?\n",
      "A: Carlos\n",
      "P: Rachel\n",
      "\n",
      "C: Peter told Chloe she should probably stop trying\n",
      "Q: Who is currently trying?\n",
      "A: Chloe\n",
      "P: Peter\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.summary(n=3, format_example_fn=format_squad_with_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 616,
   "metadata": {},
   "outputs": [],
   "source": [
    "liked = [x[0][0] for x in tg.unmask('John was told by Luke that he really likes <mask>.', beam_size=100)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = []\n",
    "data, maps = editor.template(('Who really likes {liked}?', '{male1} was told by {male2} that {male2} really likes {liked}.'), male1=editor.lexicons['male'], male2=editor.lexicons['male'], nsamples=100, verb=verbs, liked=liked, return_maps=True, remove_duplicates=True)\n",
    "labels += [m['male2'] for m in maps]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 618,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 100 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "130a2d7705454f35b5de65d29223020a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=13.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:     100\n",
      "Failure rate:   44.0%\n",
      "\n",
      "Example fails:\n",
      "C: Eric was told by Austin that Austin really likes Hunter.\n",
      "Q: Who really likes Hunter?\n",
      "A: Austin\n",
      "P: Eric\n",
      "\n",
      "C: Ethan was told by Juan that Juan really likes Jess.\n",
      "Q: Who really likes Jess?\n",
      "A: Juan\n",
      "P: Juan that Juan\n",
      "\n",
      "C: Robert was told by Ian that Ian really likes Vanessa.\n",
      "Q: Who really likes Vanessa?\n",
      "A: Ian\n",
      "P: Robert\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test = Mft(data, labels=labels)\n",
    "test.run(pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 713,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Bible', 'book', 'report', 'survey', 'poll', 'study', 'Constitution', 'data', 'bible', 'election', 'census', 'law', 'UN', 'Quran', 'evidence', 'science', 'constitution', 'Koran', 'vote', 'bill', 'film', 'Holocaust', 'research', 'Pope', 'letter', 'article', 'verdict', 'world', 'movie', 'government', 'video', 'Torah', 'song', 'moon', 'public', 'dictionary', 'president', 'referendum', 'ACA', 'eclipse', 'church', 'US', 'media', 'Church', 'pope', 'record', 'test', 'answer', 'literature', 'Prophet', 'CDC', 'FDA', 'WHO', 'scripture', 'community', 'Buddha', 'ban', 'prophet', 'statement', 'interview', 'Gospel', 'Book', 'past', 'question', 'decision']\n"
     ]
    }
   ],
   "source": [
    "print([x[0][0] for x in tg.unmask_multiple(editor.template('What does the <mask> say about {thing}?', thing=quran))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 619,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def load_squad(fold='validation'):\n",
    "    answers = []\n",
    "    data = []\n",
    "    ids = []\n",
    "    files = {\n",
    "        'validation': '/home/marcotcr/datasets/squad/dev-v1.1.json',\n",
    "        'train': '/home/marcotcr//datasets/squad/train-v1.1.json',\n",
    "        }\n",
    "    f = json.load(open(files[fold]))\n",
    "    for t in f['data']:\n",
    "        for p in t['paragraphs']:\n",
    "            context = p['context']\n",
    "            for qa in p['qas']:\n",
    "                data.append({'passage': context, 'question': qa['question'], 'id': qa['id']})\n",
    "                answers.append(set([(x['text'], x['answer_start']) for x in qa['answers']]))\n",
    "    return data, answers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 620,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, answers =  load_squad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 621,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [(x['question'], x['passage']) for x in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_name(x):\n",
    "    q, c = x\n",
    "    in_p = set()\n",
    "    not_in_p = set()\n",
    "    for n in editor.lexicons['male'][:10]:\n",
    "        if re.search(r'\\b%s\\b' % n, c):\n",
    "            in_p.add(n)\n",
    "        else:\n",
    "            not_in_p.add(n)\n",
    "    if not in_p:\n",
    "        return None\n",
    "    ret = []\n",
    "    ret_add = []\n",
    "    for p in in_p:\n",
    "        for n in not_in_p:\n",
    "            ret.append((re.sub(r'\\b%s\\b' % p, n, q), re.sub(r'\\b%s\\b' % p, n, c)))\n",
    "            ret_add.append((p, n))\n",
    "    if ret:\n",
    "        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)\n",
    "        ret = [ret[i] for i in idxs]\n",
    "        ret_add = [ret_add[i] for i in idxs]\n",
    "    return ret, ret_add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [i for i, x in enumerate(pairs) if 'John' in x[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 627,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 692,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 2058 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37654cb9177d49eca96387bc6ffd2792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=268.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:     343\n",
      "Failure rate:   1.2%\n",
      "\n",
      "Example fails:\n",
      "Q: What did John Dobson describe Newcastle as?\n",
      "P: neoclassical centre referred to as Tyneside Classical\n",
      "\n",
      "John -> Daniel\n",
      "Q: What did Daniel Dobson describe Newcastle as?\n",
      "P: neoclassical\n",
      "\n",
      "\n",
      "Q: What Doctor was first referred to as \"his secret\"?\n",
      "P: Eleventh Doctor\n",
      "\n",
      "John -> James\n",
      "Q: What Doctor was first referred to as \"his secret\"?\n",
      "P: Eleventh Doctor meets an unknown incarnation of himself\n",
      "\n",
      "John -> Joseph\n",
      "Q: What Doctor was first referred to as \"his secret\"?\n",
      "P: Eleventh Doctor meets an unknown incarnation of himself\n",
      "\n",
      "\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: both Kenia and Kegnia\n",
      "\n",
      "Joseph -> James\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: Kenia and Kegnia\n",
      "\n",
      "Joseph -> William\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: Kenia and Kegnia\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def new_eq(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    if meta:\n",
    "        p, n = meta\n",
    "        orig_pred =  re.sub(r'\\b%s\\b' % p, n, orig_pred)\n",
    "    return pred == orig_pred\n",
    "\n",
    "def format_name(x, pred, conf, label=None, meta=None):\n",
    "    ret = ''\n",
    "    if meta is not None and len(meta):\n",
    "        ret = '%s -> %s\\n' % meta\n",
    "    ret += format_squad(x, pred, conf, label, meta)\n",
    "    return ret\n",
    "    \n",
    "# tt = Expect.wrap(Expect.all(Expect.pairwise_to_group(new_eq), ignore_first=True))\n",
    "random_idxs = np.random.choice(len(pairs), 3000)\n",
    "data, meta = Perturb.perturb([pairs[i] for i in random_idxs], change_name, returns_additional=True)\n",
    "tt = Expect.pairwise(new_eq)\n",
    "test = Inv(data, expect=tt, meta=meta)\n",
    "test.run(pp)\n",
    "test.summary(n=3, format_example_fn=format_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 877,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{},\n",
       " ('John', 'Joshua'),\n",
       " ('John', 'Matthew'),\n",
       " ('John', 'William'),\n",
       " ('John', 'Christopher'),\n",
       " ('John', 'Michael'),\n",
       " ('John', 'James'),\n",
       " ('John', 'David'),\n",
       " ('John', 'Daniel'),\n",
       " ('John', 'Joseph')]"
      ]
     },
     "execution_count": 877,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.meta[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 876,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['John Fox', 'Joshua Fox', 'Matthew Fox', 'William Fox',\n",
       "       'Christopher Fox', 'Michael Fox', 'James Fox', 'David Fox',\n",
       "       'Daniel Fox', 'Joseph Fox'], dtype='<U15')"
      ]
     },
     "execution_count": 876,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.results.preds[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 864,
   "metadata": {},
   "outputs": [],
   "source": [
    "# meta[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 861,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 865,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function checklist.expect.Expect.all.<locals>.expect(xs, preds, confs, labels=None, meta=None)>"
      ]
     },
     "execution_count": 865,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 853,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('John', 'Joshua')"
      ]
     },
     "execution_count": 853,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "change_name(pairs[178])[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 802,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "671"
      ]
     },
     "execution_count": 802,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in pairs if 'John' in x[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "How is vanilla extract made?"
      ]
     },
     "execution_count": 521,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_qs[20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['John', 'Luke', 'Mark']"
      ]
     },
     "execution_count": 500,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons['male']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40430"
      ]
     },
     "execution_count": 498,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(processed_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NORP'"
      ]
     },
     "execution_count": 496,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_qs[0].ents[0][0].ent_type_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [],
   "source": [
    "# editor.template('This is {bad}',  bad=['bad', 'great', 'awesome'], return_maps=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([('This is bad', 'This is not bad'),\n",
       "  ('This is great', 'This is not great'),\n",
       "  ('This is awesome', 'This is not awesome')],\n",
       " [{'bad': 'bad'}, {'bad': 'great'}, {'bad': 'awesome'}])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "editor.template(('This is {bad}', 'This is not {bad}'),  bad=['bad', 'great', 'awesome'], return_maps=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'bad': 'This is bad', 'notbad': 'This is not bad'},\n",
       " {'bad': 'This is great', 'notbad': 'This is not great'},\n",
       " {'bad': 'This is awesome', 'notbad': 'This is not awesome'}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template({\n",
    "    'bad': 'This is {bad}',\n",
    "    'notbad': 'This is not {bad}'},  bad=['bad', 'great', 'awesome'], return_maps=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = {'bad': 'This is {bad}',\n",
    "    'notbad': ('this is not {bad}', 'this is quite {abad}')}\n",
    "b = ({'bad': 'This is {bad}', 'notbad': 'This is not {bad}'}, 'This is quite {bad}')\n",
    "c = 'This is quite {bad}.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'male': ['John', 'Luke', 'Mark'], 'female': ['Mary', 'Judy', 'Julia']}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.lexicons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is terrible, John',\n",
       " 'This is bad, John',\n",
       " 'This is terrible, Luke',\n",
       " 'This is bad, Luke',\n",
       " 'This is terrible, Mark',\n",
       " 'This is bad, Mark']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor.template('This is {bad}, {male}', bad=('terrible', 'bad'), nsamples=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "checklist",
   "language": "python",
   "name": "checklist"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
